Add driver as init container to daemonset (#828)

* add init container name replacement * Add unit tests * add driver as init container * add custom test * e2e test updates * update auth to use quay instead of dockerhub * add temp secret for auth test * convert pflex tests to create secret in memory, clean up scenarios * fix scenarios * update-image * fix auth version * fix comment * remove debug lines * lint fixes * add zoning to pflex suite * fix permissions error logged in driver pod * update unit tests --------- Co-authored-by: Evgeny Uglov <[email protected]>
dell · Dec 19, 2024 · a892d43 · a892d43
1 parent f54c83a
commit a892d43
Show file tree

Hide file tree

Showing 30 changed files with 184 additions and 135 deletions.
diff --git a/config/samples/storage_v1_csm_powerflex.yaml b/config/samples/storage_v1_csm_powerflex.yaml
@@ -70,7 +70,7 @@ spec:
           - name: HOST_PID
             value: "1"
           - name: MDM
-            value: "10.xx.xx.xx,10.xx.xx.xx" # do not add mdm value here if it is present in secret
+            value: "10.xx.xx.xx,10.xx.xx.xx"  # do not add mdm value here if it is present in secret
             # health monitor is disabled by default, refer to driver documentation before enabling it
             # Also set the env variable controller.envs.X_CSI_HEALTH_MONITOR_ENABLED  to "true".
       - name: csi-external-health-monitor-controller
@@ -198,7 +198,7 @@ spec:
         name: sdc
         envs:
           - name: MDM
-            value: "10.xx.xx.xx,10.xx.xx.xx" # provide MDM value
+            value: "10.xx.xx.xx,10.xx.xx.xx"  # provide MDM value
   modules:
     # Authorization: enable csm-authorization for RBAC
     - name: authorization

diff --git a/operatorconfig/driverconfig/powerflex/v2.13.0/node.yaml b/operatorconfig/driverconfig/powerflex/v2.13.0/node.yaml
@@ -30,6 +30,9 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["volumeattachments"]
     verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch", "update"]
   - apiGroups: ["security.openshift.io"]
     resourceNames: ["privileged"]
     resources: ["securitycontextconstraints"]
@@ -192,23 +195,48 @@ spec:
             - name: host-opt-emc-path
               mountPath: /host_opt_emc_path
       initContainers:
+        - name: mdm-container
+          image: quay.io/dell/container-storage-modules/csi-vxflexos:nightly
+          imagePullPolicy: Always
+          command: ["/csi-vxflexos.sh"]
+          args:
+            - "--array-config=/vxflexos-config/config"
+            - "--driver-config-params=/vxflexos-config-params/driver-config-params.yaml"
+          env:
+            - name: X_CSI_MODE
+              value: mdm-info
+            - name: NODENAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: vxflexos-config
+              mountPath: /vxflexos-config
+            - name: vxflexos-config-params
+              mountPath: /vxflexos-config-params
+            - name: certs
+              mountPath: /certs
+              readOnly: true
+            - name: mdm-dir
+              mountPath: /data
         - name: sdc
           securityContext:
             privileged: true
           image: dellemc/sdc:4.5.2.1
           imagePullPolicy: IfNotPresent
+          command:
+            - "/bin/sh"
+            - "-c"
+            - |
+               source /data/node_mdms.txt
+               /files/scripts/init.sh
           env:
             - name: NODENAME
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
             - name: MODE
               value: "config"
-            - name: MDM
-              valueFrom:
-                secretKeyRef:
-                  name: <DriverDefaultReleaseName>-config
-                  key: MDM
             - name: HOST_DRV_CFG_PATH
               value: /opt/emc/scaleio/sdc/bin
           volumeMounts:
@@ -224,6 +252,8 @@ spec:
               mountPath: /host_drv_cfg_path
             - name: host-opt-emc-path
               mountPath: /host_opt_emc_path
+            - name: mdm-dir
+              mountPath: /data
       volumes:
         - name: registration-dir
           hostPath:
@@ -291,6 +321,8 @@ spec:
           hostPath:
             path: /var/run
             type: Directory
+        - name: mdm-dir
+          emptyDir: {}
         - name: certs
           projected:
             sources:

diff --git a/pkg/drivers/common_test.go b/pkg/drivers/common_test.go
@@ -116,6 +116,9 @@ func csmWithTolerations(driver csmv1.DriverType, version string) csmv1.Container
 func csmForPowerFlex(customCSMName string) csmv1.ContainerStorageModule {
 	res := shared.MakeCSM(customCSMName, pFlexNS, shared.PFlexConfigVersion)
 
+	// Add driver common image
+	res.Spec.Driver.Common.Image = "driverimage"
+
 	// Add sdc initcontainer
 	res.Spec.Driver.InitContainers = []csmv1.ContainerTemplate{{
 		Name:            "sdc",

diff --git a/pkg/drivers/commonconfig.go b/pkg/drivers/commonconfig.go
@@ -315,6 +315,13 @@ func GetNode(ctx context.Context, cr csmv1.ContainerStorageModule, operatorConfi
 	for i := range initcontainers {
 		utils.ReplaceAllContainerImageApply(operatorConfig.K8sVersion, &initcontainers[i])
 		utils.UpdateInitContainerApply(cr.Spec.Driver.InitContainers, &initcontainers[i])
+		// mdm-container is exclusive to powerflex driver deamonset, will use the driver image as an init container
+		if *initcontainers[i].Name == "mdm-container" {
+			if string(cr.Spec.Driver.Common.Image) != "" {
+				image := string(cr.Spec.Driver.Common.Image)
+				initcontainers[i].Image = &image
+			}
+		}
 	}
 
 	nodeYaml.DaemonSetApplyConfig.Spec.Template.Spec.InitContainers = initcontainers

diff --git a/pkg/drivers/commonconfig_test.go b/pkg/drivers/commonconfig_test.go
@@ -127,11 +127,23 @@ func TestGetController(t *testing.T) {
 
 func TestGetNode(t *testing.T) {
 	ctx := context.Background()
+	foundInitMdm := false
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, err := GetNode(ctx, tt.csm, config, tt.driverName, tt.filename)
+			node, err := GetNode(ctx, tt.csm, config, tt.driverName, tt.filename)
 			if tt.expectedErr == "" {
 				assert.Nil(t, err)
+				initcontainers := node.DaemonSetApplyConfig.Spec.Template.Spec.InitContainers
+				for i := range initcontainers {
+					if *initcontainers[i].Name == "mdm-container" {
+						foundInitMdm = true
+						assert.Equal(t, string(tt.csm.Spec.Driver.Common.Image), *initcontainers[i].Image)
+					}
+				}
+				// if driver is powerflex, then check that mdm-container is present
+				if tt.driverName == "powerflex" {
+					assert.Equal(t, true, foundInitMdm)
+				}
 			} else {
 				assert.Containsf(t, err.Error(), tt.expectedErr, "expected error containing %q, got %s", tt.expectedErr, err)
 			}

diff --git a/pkg/modules/testdata/cr_powerflex_observability.yaml b/pkg/modules/testdata/cr_powerflex_observability.yaml
@@ -41,7 +41,7 @@ spec:
           - name: HOST_PID
             value: "1"
           - name: MDM
-            value: "10.x.x.x,10.x.x.x" # provide MDM value
+            value: "10.x.x.x,10.x.x.x"  # provide MDM value
       # health monitor is disabled by default, refer to driver documentation before enabling it
       # Also set the env variable controller.envs.X_CSI_HEALTH_MONITOR_ENABLED  to "true".
       - name: csi-external-health-monitor-controller
@@ -120,7 +120,7 @@ spec:
         name: sdc
         envs:
           - name: MDM
-            value: "10.x.x.x,10.x.x.x" # provide MDM value
+            value: "10.x.x.x,10.x.x.x"  # provide MDM value
   modules:
     # observability: allows to configure observability
     - name: observability

diff --git a/pkg/modules/testdata/cr_powerflex_observability_custom_cert.yaml b/pkg/modules/testdata/cr_powerflex_observability_custom_cert.yaml
@@ -51,7 +51,7 @@ spec:
           - name: HOST_PID
             value: "1"
           - name: MDM
-            value: "10.xx.xx.xx,10.xx.xx.xx" # do not add mdm value here if it is present in secret
+            value: "10.xx.xx.xx,10.xx.xx.xx"  # do not add mdm value here if it is present in secret
       # health monitor is disabled by default, refer to driver documentation before enabling it
       # Also set the env variable controller.envs.X_CSI_HEALTH_MONITOR_ENABLED  to "true".
       - name: csi-external-health-monitor-controller
@@ -164,7 +164,7 @@ spec:
         name: sdc
         envs:
           - name: MDM
-            value: "10.xx.xx.xx,10.xx.xx.xx" # provide MDM value
+            value: "10.xx.xx.xx,10.xx.xx.xx"  # provide MDM value
   modules:
     # observability: allows to configure observability
     - name: observability

diff --git a/pkg/modules/testdata/cr_powerflex_observability_custom_cert_missing_key.yaml b/pkg/modules/testdata/cr_powerflex_observability_custom_cert_missing_key.yaml
@@ -51,7 +51,7 @@ spec:
           - name: HOST_PID
             value: "1"
           - name: MDM
-            value: "10.xx.xx.xx,10.xx.xx.xx" # do not add mdm value here if it is present in secret
+            value: "10.xx.xx.xx,10.xx.xx.xx"  # do not add mdm value here if it is present in secret
       # health monitor is disabled by default, refer to driver documentation before enabling it
       # Also set the env variable controller.envs.X_CSI_HEALTH_MONITOR_ENABLED  to "true".
       - name: csi-external-health-monitor-controller
@@ -164,7 +164,7 @@ spec:
         name: sdc
         envs:
           - name: MDM
-            value: "10.xx.xx.xx,10.xx.xx.xx" # provide MDM value
+            value: "10.xx.xx.xx,10.xx.xx.xx"  # provide MDM value
   modules:
     # observability: allows to configure observability
     - name: observability

diff --git a/samples/storage_csm_powerflex_v2130.yaml b/samples/storage_csm_powerflex_v2130.yaml
@@ -70,7 +70,7 @@ spec:
           - name: HOST_PID
             value: "1"
           - name: MDM
-            value: "10.xx.xx.xx,10.xx.xx.xx" # do not add mdm value here if it is present in secret
+            value: "10.xx.xx.xx,10.xx.xx.xx"  # do not add mdm value here if it is present in secret
             # health monitor is disabled by default, refer to driver documentation before enabling it
             # Also set the env variable controller.envs.X_CSI_HEALTH_MONITOR_ENABLED  to "true".
             # Default monitor-interval: 60s
@@ -199,7 +199,7 @@ spec:
         name: sdc
         envs:
           - name: MDM
-            value: "10.xx.xx.xx,10.xx.xx.xx" # provide MDM value
+            value: "10.xx.xx.xx,10.xx.xx.xx"  # provide MDM value
   modules:
     # Authorization: enable csm-authorization for RBAC
     - name: authorization

diff --git a/tests/config/driverconfig/powerflex/v2.13.0/node.yaml b/tests/config/driverconfig/powerflex/v2.13.0/node.yaml
@@ -183,23 +183,50 @@ spec:
             - name: udev-d
               mountPath: /rules.d
       initContainers:
+        - name: mdm-container
+          image: quay.io/dell/container-storage-modules/csi-vxflexos:nightly
+          imagePullPolicy: Always
+          command: ["/csi-vxflexos.sh"]
+          args:
+            - "--array-config=/vxflexos-config/config"
+            - "--driver-config-params=/vxflexos-config-params/driver-config-params.yaml"
+          env:
+            - name: X_CSI_MODE
+              value: mdm-info
+            - name: NODENAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: vxflexos-config
+              mountPath: /vxflexos-config
+            - name: vxflexos-config-params
+              mountPath: /vxflexos-config-params
+            - name: certs
+              mountPath: /certs
+              readOnly: true
+            - name: mdm-dir
+              mountPath: /data
         - name: sdc
           securityContext:
             privileged: true
           image: dellemc/sdc:4.5.2.1
           imagePullPolicy: IfNotPresent
+          command:
+            - "/bin/sh"
+            - "-c"
+            - |
+               echo "Inside sdc init-container"
+               source /data/node_mdms.txt
+               env
+               /files/scripts/init.sh
           env:
             - name: NODENAME
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
             - name: MODE
               value: "config"
-            - name: MDM
-              valueFrom:
-                secretKeyRef:
-                  name: <DriverDefaultReleaseName>-config
-                  key: MDM
             - name: HOST_DRV_CFG_PATH
               value: /opt/emc/scaleio/sdc/bin
           volumeMounts:

diff --git a/tests/e2e/testfiles/authorization-templates/storage_csm_authorization_v1_proxy_server.yaml b/tests/e2e/testfiles/authorization-templates/storage_csm_authorization_v1_proxy_server.yaml
@@ -9,7 +9,7 @@ spec:
     - name: authorization-proxy-server
       # enable: Enable/Disable csm-authorization
       enabled: true
-      configVersion: v1.11.0
+      configVersion: v1.13.0
       forceRemoveModule: true
       components:
         # For Kubernetes Container Platform only
@@ -30,12 +30,12 @@ spec:
         - name: proxy-server
           # enable: Enable/Disable csm-authorization proxy server
           enabled: true
-          proxyService: dellemc/csm-authorization-proxy:v1.11.0
-          tenantService: dellemc/csm-authorization-tenant:v1.11.0
-          roleService: dellemc/csm-authorization-role:v1.11.0
-          storageService: dellemc/csm-authorization-storage:v1.11.0
+          proxyService: quay.io/dell/container-storage-modules/csm-authorization-proxy:v1-nightly
+          tenantService: quay.io/dell/container-storage-modules/csm-authorization-tenant:v1-nightly
+          roleService: quay.io/dell/container-storage-modules/csm-authorization-role:v1-nightly
+          storageService: quay.io/dell/container-storage-modules/csm-authorization-storage:v1-nightly
           opa: openpolicyagent/opa
-          opaKubeMgmt: openpolicyagent/kube-mgmt:0.11
+          opaKubeMgmt: openpolicyagent/kube-mgmt:8.5.7
           # certificate: base64-encoded certificate for cert/private-key pair -- add certificate here to use custom certificates
           # for self-signed certs, leave empty string
           # Allowed values: string

diff --git a/...estfiles/authorization-templates/storage_csm_authorization_v1_proxy_server_n_minus_1.yaml b/...estfiles/authorization-templates/storage_csm_authorization_v1_proxy_server_n_minus_1.yaml
@@ -30,12 +30,12 @@ spec:
         - name: proxy-server
           # enable: Enable/Disable csm-authorization proxy server
           enabled: true
-          proxyService: dellemc/csm-authorization-proxy:v1.12.0
-          tenantService: dellemc/csm-authorization-tenant:v1.12.0
-          roleService: dellemc/csm-authorization-role:v1.12.0
-          storageService: dellemc/csm-authorization-storage:v1.12.0
+          proxyService: quay.io/dell/container-storage-modules/csm-authorization-proxy:v1.12.0
+          tenantService: quay.io/dell/container-storage-modules/csm-authorization-tenant:v1.12.0
+          roleService: quay.io/dell/container-storage-modules/csm-authorization-role:v1.12.0
+          storageService: quay.io/dell/container-storage-modules/csm-authorization-storage:v1.12.0
           opa: openpolicyagent/opa
-          opaKubeMgmt: openpolicyagent/kube-mgmt:0.11
+          opaKubeMgmt: openpolicyagent/kube-mgmt:8.5.7
           # certificate: base64-encoded certificate for cert/private-key pair -- add certificate here to use custom certificates
           # for self-signed certs, leave empty string
           # Allowed values: string

diff --git a/.../testfiles/authorization-templates/storage_csm_authorization_v1_proxy_server_no_cert.yaml b/.../testfiles/authorization-templates/storage_csm_authorization_v1_proxy_server_no_cert.yaml
@@ -30,10 +30,10 @@ spec:
         - name: proxy-server
           # enable: Enable/Disable csm-authorization proxy server
           enabled: true
-          proxyService: quay.io/dell/container-storage-modules/csm-authorization-proxy:v1.13.0
-          tenantService: quay.io/dell/container-storage-modules/csm-authorization-tenant:v1.13.0
-          roleService: quay.io/dell/container-storage-modules/csm-authorization-role:v1.13.0
-          storageService: quay.io/dell/container-storage-modules/csm-authorization-storage:v1.13.0
+          proxyService: quay.io/dell/container-storage-modules/csm-authorization-proxy:v1-nightly
+          tenantService: quay.io/dell/container-storage-modules/csm-authorization-tenant:v1-nightly
+          roleService: quay.io/dell/container-storage-modules/csm-authorization-role:v1-nightly
+          storageService: quay.io/dell/container-storage-modules/csm-authorization-storage:v1-nightly
           opa: docker.io/openpolicyagent/opa:latest
           opaKubeMgmt: docker.io/openpolicyagent/kube-mgmt:8.5.7
           # certificate: base64-encoded certificate for cert/private-key pair -- add certificate here to use custom certificates