diff --git a/cmd/smbplugin/main.go b/cmd/smbplugin/main.go index 788a04e50c9..3562a653832 100644 --- a/cmd/smbplugin/main.go +++ b/cmd/smbplugin/main.go @@ -47,6 +47,7 @@ var ( volStatsCacheExpireInMinutes = flag.Int("vol-stats-cache-expire-in-minutes", 10, "The cache expire time in minutes for volume stats cache") krb5CacheDirectory = flag.String("krb5-cache-directory", smb.DefaultKrb5CacheDirectory, "The directory for kerberos cache") krb5Prefix = flag.String("krb5-prefix", smb.DefaultKrb5CCName, "The prefix for kerberos cache") + defaultOnDeletePolicy = flag.String("default-ondelete-policy", "", "default policy for deleting subdirectory when deleting a volume") ) func main() { @@ -78,6 +79,7 @@ func handle() { VolStatsCacheExpireInMinutes: *volStatsCacheExpireInMinutes, Krb5CacheDirectory: *krb5CacheDirectory, Krb5Prefix: *krb5Prefix, + DefaultOnDeletePolicy: *defaultOnDeletePolicy, } driver := smb.NewDriver(&driverOptions) driver.Run(*endpoint, *kubeconfig, false) diff --git a/docs/driver-parameters.md b/docs/driver-parameters.md index d4ea7febda8..21076f34482 100644 --- a/docs/driver-parameters.md +++ b/docs/driver-parameters.md @@ -7,6 +7,7 @@ Name | Meaning | Available Value | Mandatory | Default value --- | --- | --- | --- | --- source | Samba Server address | `//smb-server-address/sharename`
([Azure File](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction) format: `//accountname.file.core.windows.net/filesharename`) | Yes | subDir | sub directory under smb share | | No | if sub directory does not exist, this driver would create a new one +onDelete | when volume is deleted, keep the directory if it's `retain` | `delete`(default), `retain`, `archive` | No | `delete` csi.storage.k8s.io/provisioner-secret-name | secret name that stores `username`, `password`(`domain` is optional); if secret is provided, driver will create a sub directory with PV name under `source` | existing secret name | No | csi.storage.k8s.io/provisioner-secret-namespace | namespace where the secret is | existing secret namespace | No | csi.storage.k8s.io/node-stage-secret-name | secret name that stores `username`, `password`(`domain` is optional) | existing secret name | Yes | @@ -35,10 +36,6 @@ kubectl create secret generic smbcreds --from-literal username=USERNAME --from-l ``` ### Kerberos ticket support for Linux - - - - #### These are the conditions that must be met: - Kerberos support should be set up and cifs-utils must be installed on every node. - The directory /var/lib/kubelet/kerberos/ needs to exist, and it will hold kerberos credential cache files for various users. diff --git a/pkg/smb/controllerserver.go b/pkg/smb/controllerserver.go index 74015ab83e4..aa61126f9dc 100644 --- a/pkg/smb/controllerserver.go +++ b/pkg/smb/controllerserver.go @@ -47,6 +47,8 @@ type smbVolume struct { size int64 // pv name when subDir is not empty uuid string + // on delete action + onDelete string } // Ordering of elements in the CSI volume id. @@ -55,6 +57,7 @@ const ( idSource = iota idSubDir idUUID + idOnDelete totalIDElements // Always last ) @@ -74,7 +77,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) if parameters == nil { parameters = make(map[string]string) } - smbVol, err := newSMBVolume(name, reqCapacity, parameters) + smbVol, err := newSMBVolume(name, reqCapacity, parameters, d.defaultOnDeletePolicy) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -158,9 +161,13 @@ func (d *Driver) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) } } - if len(req.GetSecrets()) > 0 { - klog.V(2).Infof("begin to delete subdirectory since secret is provided") - // Mount smb base share so we can delete the subdirectory + if smbVol.onDelete == "" { + smbVol.onDelete = d.defaultOnDeletePolicy + } + + if len(req.GetSecrets()) > 0 && !strings.EqualFold(smbVol.onDelete, retain) { + klog.V(2).Infof("begin to delete or archive subdirectory since secret is provided") + // mount smb base share so we can delete or archive the subdirectory if err = d.internalMount(ctx, smbVol, volCap, secrets); err != nil { return nil, status.Errorf(codes.Internal, "failed to mount smb server: %v", err.Error()) } @@ -170,11 +177,20 @@ func (d *Driver) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) } }() - // Delete subdirectory under base-dir internalVolumePath := getInternalVolumePath(d.workingMountDir, smbVol) - klog.V(2).Infof("Removing subdirectory at %v", internalVolumePath) - if err = os.RemoveAll(internalVolumePath); err != nil { - return nil, status.Errorf(codes.Internal, "failed to delete subdirectory: %v", err.Error()) + if strings.EqualFold(smbVol.onDelete, archive) { + archivedInternalVolumePath := filepath.Join(getInternalMountPath(d.workingMountDir, smbVol), "archived-"+smbVol.subDir) + + // archive subdirectory under base-dir + klog.V(2).Infof("archiving subdirectory %s --> %s", internalVolumePath, archivedInternalVolumePath) + if err = os.Rename(internalVolumePath, archivedInternalVolumePath); err != nil { + return nil, status.Errorf(codes.Internal, "archive subdirectory(%s, %s) failed with %v", internalVolumePath, archivedInternalVolumePath, err.Error()) + } + } else { + klog.V(2).Infof("Removing subdirectory at %v", internalVolumePath) + if err = os.RemoveAll(internalVolumePath); err != nil { + return nil, status.Errorf(codes.Internal, "failed to delete subdirectory: %v", err.Error()) + } } } else { klog.V(2).Infof("DeleteVolume(%s) does not delete subdirectory", volumeID) @@ -345,6 +361,9 @@ func getVolumeIDFromSmbVol(vol *smbVolume) string { idElements[idSource] = strings.Trim(vol.source, "/") idElements[idSubDir] = strings.Trim(vol.subDir, "/") idElements[idUUID] = vol.uuid + if strings.EqualFold(vol.onDelete, retain) || strings.EqualFold(vol.onDelete, archive) { + idElements[idOnDelete] = vol.onDelete + } return strings.Join(idElements, separator) } @@ -361,8 +380,8 @@ func getInternalMountPath(workingMountDir string, vol *smbVolume) string { } // Convert VolumeCreate parameters to an smbVolume -func newSMBVolume(name string, size int64, params map[string]string) (*smbVolume, error) { - var source, subDir string +func newSMBVolume(name string, size int64, params map[string]string, defaultOnDeletePolicy string) (*smbVolume, error) { + var source, subDir, onDelete string subDirReplaceMap := map[string]string{} // validate parameters (case-insensitive). @@ -372,6 +391,8 @@ func newSMBVolume(name string, size int64, params map[string]string) (*smbVolume source = v case subDirField: subDir = v + case paramOnDelete: + onDelete = v case pvcNamespaceKey: subDirReplaceMap[pvcNamespaceMetadata] = v case pvcNameKey: @@ -400,6 +421,16 @@ func newSMBVolume(name string, size int64, params map[string]string) (*smbVolume // make volume id unique if subDir is provided vol.uuid = name } + + if err := validateOnDeleteValue(onDelete); err != nil { + return nil, err + } + + vol.onDelete = defaultOnDeletePolicy + if onDelete != "" { + vol.onDelete = onDelete + } + vol.id = getVolumeIDFromSmbVol(vol) return vol, nil } @@ -447,6 +478,9 @@ func getSmbVolFromID(id string) (*smbVolume, error) { if len(segments) >= 3 { vol.uuid = segments[2] } + if len(segments) >= 4 { + vol.onDelete = segments[3] + } return vol, nil } diff --git a/pkg/smb/controllerserver_test.go b/pkg/smb/controllerserver_test.go index 39f1546c2ef..d4d172d1b23 100644 --- a/pkg/smb/controllerserver_test.go +++ b/pkg/smb/controllerserver_test.go @@ -35,7 +35,7 @@ import ( const ( testServer = "test-server/baseDir" testCSIVolume = "test-csi" - testVolumeID = "test-server/baseDir#test-csi#" + testVolumeID = "test-server/baseDir#test-csi##" ) func TestControllerGetCapabilities(t *testing.T) { @@ -448,6 +448,7 @@ func TestGetSmbVolFromID(t *testing.T) { source string subDir string uuid string + onDelete string expectErr bool }{ { @@ -486,6 +487,24 @@ func TestGetSmbVolFromID(t *testing.T) { uuid: "pvc-4729891a-f57e-4982-9c60-e9884af1be2f", expectErr: false, }, + { + desc: "valid request nested ondelete retain", + volumeID: "smb-server.default.svc.cluster.local/share#subdir#pvc-4729891a-f57e-4982-9c60-e9884af1be2f#retain", + source: "//smb-server.default.svc.cluster.local/share", + subDir: "subdir", + uuid: "pvc-4729891a-f57e-4982-9c60-e9884af1be2f", + onDelete: "retain", + expectErr: false, + }, + { + desc: "valid request nested ondelete archive", + volumeID: "smb-server.default.svc.cluster.local/share#subdir#pvc-4729891a-f57e-4982-9c60-e9884af1be2f#archive", + source: "//smb-server.default.svc.cluster.local/share", + subDir: "subdir", + uuid: "pvc-4729891a-f57e-4982-9c60-e9884af1be2f", + onDelete: "archive", + expectErr: false, + }, { desc: "incorrect volume id", volumeID: "smb-server.default.svc.cluster.local/share", @@ -503,6 +522,7 @@ func TestGetSmbVolFromID(t *testing.T) { assert.Equal(t, smbVolume.source, test.source) assert.Equal(t, smbVolume.subDir, test.subDir) assert.Equal(t, smbVolume.uuid, test.uuid) + assert.Equal(t, smbVolume.onDelete, test.onDelete) assert.Nil(t, err) } else { assert.NotNil(t, err) @@ -523,7 +543,7 @@ func TestGetVolumeIDFromSmbVol(t *testing.T) { source: "//smb-server.default.svc.cluster.local/share", subDir: "subdir", }, - result: "smb-server.default.svc.cluster.local/share#subdir#", + result: "smb-server.default.svc.cluster.local/share#subdir##", }, { desc: "volume with uuid", @@ -532,14 +552,24 @@ func TestGetVolumeIDFromSmbVol(t *testing.T) { subDir: "subdir", uuid: "uuid", }, - result: "smb-server.default.svc.cluster.local/share#subdir#uuid", + result: "smb-server.default.svc.cluster.local/share#subdir#uuid#", }, { desc: "volume without subdir", vol: &smbVolume{ source: "//smb-server.default.svc.cluster.local/share", }, - result: "smb-server.default.svc.cluster.local/share##", + result: "smb-server.default.svc.cluster.local/share###", + }, + { + desc: "volume with nested onDelete retain", + vol: &smbVolume{ + source: "//smb-server.default.svc.cluster.local/share", + subDir: "subdir", + uuid: "uuid", + onDelete: "retain", + }, + result: "smb-server.default.svc.cluster.local/share#subdir#uuid#retain", }, } @@ -605,7 +635,7 @@ func TestNewSMBVolume(t *testing.T) { "subDir": "subdir", }, expectVol: &smbVolume{ - id: "smb-server.default.svc.cluster.local/share#subdir#pv-name", + id: "smb-server.default.svc.cluster.local/share#subdir#pv-name#", source: "//smb-server.default.svc.cluster.local/share", subDir: "subdir", size: 100, @@ -624,7 +654,7 @@ func TestNewSMBVolume(t *testing.T) { pvNameKey: "pvname", }, expectVol: &smbVolume{ - id: "smb-server.default.svc.cluster.local/share#subdir-pvcname-pvcnamespace-pvname#pv-name", + id: "smb-server.default.svc.cluster.local/share#subdir-pvcname-pvcnamespace-pvname#pv-name#", source: "//smb-server.default.svc.cluster.local/share", subDir: "subdir-pvcname-pvcnamespace-pvname", size: 100, @@ -639,7 +669,7 @@ func TestNewSMBVolume(t *testing.T) { "source": "//smb-server.default.svc.cluster.local/share", }, expectVol: &smbVolume{ - id: "smb-server.default.svc.cluster.local/share#pv-name#", + id: "smb-server.default.svc.cluster.local/share#pv-name##", source: "//smb-server.default.svc.cluster.local/share", subDir: "pv-name", size: 200, @@ -661,7 +691,7 @@ func TestNewSMBVolume(t *testing.T) { } for _, test := range cases { - vol, err := newSMBVolume(test.name, test.size, test.params) + vol, err := newSMBVolume(test.name, test.size, test.params, "") if !reflect.DeepEqual(err, test.expectErr) { t.Errorf("[test: %s] Unexpected error: %v, expected error: %v", test.desc, err, test.expectErr) } diff --git a/pkg/smb/smb.go b/pkg/smb/smb.go index ddc7427d79a..e2550befe38 100644 --- a/pkg/smb/smb.go +++ b/pkg/smb/smb.go @@ -17,6 +17,7 @@ limitations under the License. package smb import ( + "fmt" "strings" "time" @@ -39,6 +40,7 @@ const ( subDirField = "subdir" domainField = "domain" mountOptionsField = "mountoptions" + paramOnDelete = "ondelete" defaultDomainName = "AZURE" pvcNameKey = "csi.storage.k8s.io/pvc/name" pvcNamespaceKey = "csi.storage.k8s.io/pvc/namespace" @@ -48,8 +50,12 @@ const ( pvNameMetadata = "${pv.metadata.name}" DefaultKrb5CCName = "krb5cc_" DefaultKrb5CacheDirectory = "/var/lib/kubelet/kerberos/" + retain = "retain" + archive = "archive" ) +var supportedOnDeleteValues = []string{"", "delete", retain, archive} + // DriverOptions defines driver parameters specified in driver deployment type DriverOptions struct { NodeID string @@ -61,6 +67,7 @@ type DriverOptions struct { VolStatsCacheExpireInMinutes int Krb5CacheDirectory string Krb5Prefix string + DefaultOnDeletePolicy string } // Driver implements all interfaces of CSI drivers @@ -78,6 +85,7 @@ type Driver struct { removeSMBMappingDuringUnmount bool krb5CacheDirectory string krb5Prefix string + defaultOnDeletePolicy string } // NewDriver Creates a NewCSIDriver object. Assumes vendor version is equal to driver version & @@ -208,3 +216,13 @@ func replaceWithMap(str string, m map[string]string) string { } return str } + +func validateOnDeleteValue(onDelete string) error { + for _, v := range supportedOnDeleteValues { + if strings.EqualFold(v, onDelete) { + return nil + } + } + + return fmt.Errorf("invalid value %s for OnDelete, supported values are %v", onDelete, supportedOnDeleteValues) +} diff --git a/pkg/smb/smb_test.go b/pkg/smb/smb_test.go index 056480cb9d6..f030d0b68d7 100644 --- a/pkg/smb/smb_test.go +++ b/pkg/smb/smb_test.go @@ -17,6 +17,7 @@ limitations under the License. package smb import ( + "fmt" "os" "path/filepath" "reflect" @@ -281,3 +282,59 @@ func TestReplaceWithMap(t *testing.T) { } } } + +func TestValidateOnDeleteValue(t *testing.T) { + tests := []struct { + desc string + onDelete string + expected error + }{ + { + desc: "empty value", + onDelete: "", + expected: nil, + }, + { + desc: "delete value", + onDelete: "delete", + expected: nil, + }, + { + desc: "retain value", + onDelete: "retain", + expected: nil, + }, + { + desc: "Retain value", + onDelete: "Retain", + expected: nil, + }, + { + desc: "Delete value", + onDelete: "Delete", + expected: nil, + }, + { + desc: "Archive value", + onDelete: "Archive", + expected: nil, + }, + { + desc: "archive value", + onDelete: "archive", + expected: nil, + }, + { + desc: "invalid value", + onDelete: "invalid", + expected: fmt.Errorf("invalid value %s for OnDelete, supported values are %v", "invalid", supportedOnDeleteValues), + }, + } + + for _, test := range tests { + result := validateOnDeleteValue(test.onDelete) + if !reflect.DeepEqual(result, test.expected) { + t.Errorf("test[%s]: unexpected output: %v, expected result: %v", test.desc, result, test.expected) + } + } +} diff --git a/test/e2e/dynamic_provisioning_test.go b/test/e2e/dynamic_provisioning_test.go index 86b7d63212f..bf64af8d7fa 100644 --- a/test/e2e/dynamic_provisioning_test.go +++ b/test/e2e/dynamic_provisioning_test.go @@ -350,4 +350,70 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { } test.Run(ctx, cs, ns) }) + + ginkgo.It("should create a volume on demand with retaining subdir on delete [smb.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + pods := []testsuites.PodDetails{ + { + Cmd: convertToPowershellCommandIfNecessary("echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data"), + Volumes: []testsuites.VolumeDetails{ + { + ClaimSize: "10Gi", + MountOptions: []string{ + "dir_mode=0777", + "file_mode=0777", + "uid=0", + "gid=0", + "mfsymlinks", + "cache=strict", + "nosharesock", + }, + VolumeMount: testsuites.VolumeMountDetails{ + NameGenerate: "test-volume-", + MountPathGenerate: "/mnt/test-", + }, + }, + }, + IsWindows: isWindowsCluster, + }, + } + test := testsuites.DynamicallyProvisionedCmdVolumeTest{ + CSIDriver: testDriver, + Pods: pods, + StorageClassParameters: retainStorageClassParameters, + } + test.Run(ctx, cs, ns) + }) + + ginkgo.It("should create a volume on demand with archive subdir on archive [smb.csi.k8s.io]", func(ctx ginkgo.SpecContext) { + pods := []testsuites.PodDetails{ + { + Cmd: convertToPowershellCommandIfNecessary("echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data"), + Volumes: []testsuites.VolumeDetails{ + { + ClaimSize: "10Gi", + MountOptions: []string{ + "dir_mode=0777", + "file_mode=0777", + "uid=0", + "gid=0", + "mfsymlinks", + "cache=strict", + "nosharesock", + }, + VolumeMount: testsuites.VolumeMountDetails{ + NameGenerate: "test-volume-", + MountPathGenerate: "/mnt/test-", + }, + }, + }, + IsWindows: isWindowsCluster, + }, + } + test := testsuites.DynamicallyProvisionedCmdVolumeTest{ + CSIDriver: testDriver, + Pods: pods, + StorageClassParameters: archiveStorageClassParameters, + } + test.Run(ctx, cs, ns) + }) }) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index d3fe2ae522f..872790eefb6 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -62,6 +62,22 @@ var ( "csi.storage.k8s.io/node-stage-secret-name": "smbcreds", "csi.storage.k8s.io/node-stage-secret-namespace": "default", } + retainStorageClassParameters = map[string]string{ + "source": "//smb-server.default.svc.cluster.local/share", + "csi.storage.k8s.io/provisioner-secret-name": "smbcreds", + "csi.storage.k8s.io/provisioner-secret-namespace": "default", + "csi.storage.k8s.io/node-stage-secret-name": "smbcreds", + "csi.storage.k8s.io/node-stage-secret-namespace": "default", + "onDelete": "retain", + } + archiveStorageClassParameters = map[string]string{ + "source": "//smb-server.default.svc.cluster.local/share", + "csi.storage.k8s.io/provisioner-secret-name": "smbcreds", + "csi.storage.k8s.io/provisioner-secret-namespace": "default", + "csi.storage.k8s.io/node-stage-secret-name": "smbcreds", + "csi.storage.k8s.io/node-stage-secret-namespace": "default", + "onDelete": "archive", + } noProvisionerSecretStorageClassParameters = map[string]string{ "source": "//smb-server.default.svc.cluster.local/share", "csi.storage.k8s.io/node-stage-secret-name": "smbcreds",