Merge pull request #30 from ThinkParQ/iamjoemccormick/support-volume-…

…expansion

Add support for expanding volumes

Makefile

@@ -58,10 +58,20 @@ bin/chwrap.tar: build-chwrap cmd/chwrap/chwrap.sh
 		fi; \
 	done	
 
-# The beegfs-csi-driver container requires chwrap to be built and included, so we build it anytime
-# container, push, or push-multiarch are made. Additional prerequisites and the recipes for
-# container and push are defined in release-tools/build.make.
-container: build-chwrap bin/chwrap.tar
+
+# This target is mainly used for development to first rebuild the driver binary before building the
+# container for local testing. Since the beegfs-csi-driver container requires chwrap to be built and
+# included, we also build it anytime container, push, or push-multiarch are made. Additional
+# prerequisites and the recipes for container and push are defined in release-tools/build.make.
+#
+# IMPORTANT: Because the release tool's build.make file specifies BUILD_PLATFORMS= and cannot be
+# modified, a default set of build platforms cannot be specified in this file and thus must be
+# always provided on the command line otherwise the resulting files will not work correctly with how
+# the project's Dockerfile expects them to be named.
+#
+# For ARM: `make BUILD_PLATFORMS="linux arm64 arm64 arm64" container` For x86: `make
+# BUILD_PLATFORMS="linux amd64 amd64 amd64" container`
+container: all
 push-multiarch: build-chwrap bin/chwrap.tar
 push: container  # not explicitly executed in release-tools/build.make
 

README.md

@@ -204,7 +204,7 @@ deployment guide](operator/README.md).
      ```
 4. Run `kubectl apply -k deploy/k8s/overlays/default`. Note by default the
    beegfs-csi-driver image will be pulled from
-   [DockerHub](https://hub.docker.com/r/netapp/beegfs-csi-driver).
+   [GitHub Container Registry](https://github.com/ThinkParQ/beegfs-csi-driver/pkgs/container/beegfs-csi-driver).
 5. Verify all components are installed and operational: `kubectl get pods -n
    beegfs-csi`.
 

deploy/k8s/bases/csi-beegfs-controller.yaml

@@ -48,6 +48,27 @@ spec:
             requests:
               cpu: 80m
               memory: 24Mi
+        - name: csi-resizer
+          image: registry.k8s.io/sig-storage/csi-resizer:v1.11.1
+          args:
+            - "--csi-address=/csi/csi.sock"
+            - -v=$(LOG_LEVEL)            
+          securityContext:
+            # On SELinux enabled systems, a non-privileged sidecar container cannot access the unix domain socket
+            # created by the privileged driver container.
+            privileged: true            
+          env:
+            - name: LOG_LEVEL
+              value: "3"             
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+          resources:
+            limits:
+              memory: 500Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi           
         - name: beegfs
           image: ghcr.io/thinkparq/beegfs-csi-driver:v1.6.0
           args:

deploy/k8s/bases/csi-beegfs-rbac.yaml

@@ -24,10 +24,10 @@ metadata:
 rules:
   - apiGroups: [""]
     resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "delete"]
+    verbs: ["get", "list", "watch", "create", "delete", "patch"]
   - apiGroups: [""]
     resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch", "update"]
+    verbs: ["get", "list", "watch", "update", "patch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["storageclasses"]
     verbs: ["get", "list", "watch"]
@@ -40,6 +40,12 @@ rules:
   - apiGroups: [""]
     resources: ["nodes"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["patch"]     
 
 ---
 

deploy/k8s/deploy.go

@@ -42,6 +42,7 @@ const (
 	ContainerNameBeegfsCsiDriver        = "beegfs"
 	ContainerNameCsiNodeDriverRegistrar = "node-driver-registrar"
 	ContainerNameCsiProvisioner         = "csi-provisioner"
+	ContainerNameCsiResizer             = "csi-resizer"
 	ContainerNameLivenessProbe          = "liveness-probe"
 )
 

deploy/k8s/overlays/default/kustomization.yaml

@@ -43,6 +43,9 @@ secretGenerator:
 #   - name: k8s.gcr.io/sig-storage/csi-provisioner
 #     newName:
 #     newTag:
+#   - name: k8s.gcr.io/sig-storage/csi-resizer
+#     newName:
+#     newTag:
 #   - name: k8s.gcr.io/sig-storage/livenessprobe
 #     newName:
 #     newTag:

docs/usage.md

@@ -69,6 +69,13 @@ any POSIX) filesystem. The driver does provide integration with BeeGFS permissio
 and quotas which provides ways to limit the capacity consumed by containers. For
 more details refer to the documentation on [Quotas](quotas.md).
 
+Starting with v1.7.0 the driver also supports [volume
+expansion](https://kubernetes-csi.github.io/docs/volume-expansion.html), which is useful for
+instances where the persistent volume claim request size has meaning for the application. As with
+the initial capacity request, the size of the PVC and PV are simply updated in the Kubernetes API to
+reflect the requested new capacity, and there are no checks there is actually sufficient space
+available to satisfy the requested capacity.
+
 <a name="static-vs-dynamic-provisioning"></a>
 ### Static vs Dynamic Provisioning
 

examples/k8s/dyn/dyn-sc.yaml

@@ -20,4 +20,4 @@ parameters:
   # permissions/mode: "0644"
 reclaimPolicy: Delete
 volumeBindingMode: Immediate
-allowVolumeExpansion: false
+allowVolumeExpansion: true

operator/Makefile

@@ -6,10 +6,12 @@
 VERSION ?= 1.6.0
 
 # BUILD_PLATFORMS contains a set of tuples [os arch buildx_platform suffix base_image addon_image]
-# separated by semicolon. An empty variable or empty entry (= just a
-# semicolon) builds for the default platform of the current Go
-# toolchain. This approach was adapted from the CSI driver release-tools.1
-BUILD_PLATFORMS =
+# separated by semicolon. All supported architectures are listed here so other targets used for
+# development such as docker-build work correctly even when not building multi-arch images. This is
+# because the Dockerfile expects all binaries will have an architecture suffix. Note for the
+# official images/binaries published using GitHub actions, the default here is overridden by the
+# RELEASE_TOOLS_BUILD_PLATFORMS variable (see the notes there for why).
+BUILD_PLATFORMS ?= linux amd64 amd64 amd64;linux arm64 arm64 arm64
 
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")

operator/README.md

@@ -209,6 +209,9 @@ spec:
     csiProvisioner:
       image: some.registry/sig-storage/csi-provisioner
       tag: # Changing this tag is not supported.
+    csiResizer:
+      image: some.registry/sig-storage/csi-resizer
+      tag: # Changing this tag is not supported.      
     livenessProbe:
       image: some.registry/sig-storage/livenessprobe
       tag: # Changing this tag is not supported.

operator/api/v1/beegfsdriver_types.go

@@ -113,6 +113,9 @@ type ContainerImageOverrides struct {
 	// Defaults to registry.k8s.io/sig-storage/csi-provisioner:<the most current version at operator release>.
 	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="CSI Provisioner"
 	CsiProvisioner ContainerImageOverride `json:"csiProvisioner"`
+	// Defaults to registry.k8s.io/sig-storage/csi-resizer:<the most current version at operator release>.
+	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="CSI Resizer"
+	CsiResizer ContainerImageOverride `json:"csiResizer"`
 	// Defaults to registry.k8s.io/sig-storage/livenessprobe:<the most current version at operator release>.
 	//+operator-sdk:csv:customresourcedefinitions:type=spec
 	LivenessProbe ContainerImageOverride `json:"livenessProbe"`

operator/bundle/manifests/beegfs-csi-driver-operator.clusterserviceversion.yaml

@@ -128,6 +128,17 @@ spec:
       - description: A tag (e.g. v2.2.2 or latest).
         displayName: Tag
         path: containerImageOverrides.csiProvisioner.tag
+      - description: Defaults to registry.k8s.io/sig-storage/csi-resizer:<the most
+          current version at operator release>.
+        displayName: CSI Resizer
+        path: containerImageOverrides.csiResizer
+      - description: A combination of registry and image (e.g. registry.k8s.io/csi-provisioner
+          or ghcr.io/thinkparq/beegfs-csi-driver).
+        displayName: Image
+        path: containerImageOverrides.csiResizer.image
+      - description: A tag (e.g. v2.2.2 or latest).
+        displayName: Tag
+        path: containerImageOverrides.csiResizer.tag
       - description: Defaults to registry.k8s.io/sig-storage/livenessprobe:<the most
           current version at operator release>.
         displayName: Liveness Probe
@@ -472,8 +483,15 @@ spec:
           verbs:
           - get
           - list
+          - patch
           - update
           - watch
+        - apiGroups:
+          - ""
+          resources:
+          - persistentvolumeclaims/status
+          verbs:
+          - patch
         - apiGroups:
           - ""
           resources:
@@ -483,6 +501,15 @@ spec:
           - delete
           - get
           - list
+          - patch
+          - watch
+        - apiGroups:
+          - ""
+          resources:
+          - pods
+          verbs:
+          - get
+          - list
           - watch
         - apiGroups:
           - ""

operator/bundle/manifests/beegfs.csi.netapp.com_beegfsdrivers.yaml

@@ -87,6 +87,18 @@ spec:
                         description: A tag (e.g. v2.2.2 or latest).
                         type: string
                     type: object
+                  csiResizer:
+                    description: Defaults to registry.k8s.io/sig-storage/csi-resizer:<the
+                      most current version at operator release>.
+                    properties:
+                      image:
+                        description: A combination of registry and image (e.g. registry.k8s.io/csi-provisioner
+                          or ghcr.io/thinkparq/beegfs-csi-driver).
+                        type: string
+                      tag:
+                        description: A tag (e.g. v2.2.2 or latest).
+                        type: string
+                    type: object
                   livenessProbe:
                     description: Defaults to registry.k8s.io/sig-storage/livenessprobe:<the
                       most current version at operator release>.

operator/config/crd/bases/beegfs.csi.netapp.com_beegfsdrivers.yaml

@@ -81,6 +81,18 @@ spec:
                         description: A tag (e.g. v2.2.2 or latest).
                         type: string
                     type: object
+                  csiResizer:
+                    description: Defaults to registry.k8s.io/sig-storage/csi-resizer:<the
+                      most current version at operator release>.
+                    properties:
+                      image:
+                        description: A combination of registry and image (e.g. registry.k8s.io/csi-provisioner
+                          or ghcr.io/thinkparq/beegfs-csi-driver).
+                        type: string
+                      tag:
+                        description: A tag (e.g. v2.2.2 or latest).
+                        type: string
+                    type: object
                   livenessProbe:
                     description: Defaults to registry.k8s.io/sig-storage/livenessprobe:<the
                       most current version at operator release>.

operator/config/manifests/bases/beegfs-csi-driver-operator.clusterserviceversion.yaml

@@ -68,6 +68,17 @@ spec:
       - description: A tag (e.g. v2.2.2 or latest).
         displayName: Tag
         path: containerImageOverrides.csiProvisioner.tag
+      - description: Defaults to registry.k8s.io/sig-storage/csi-resizer:<the most
+          current version at operator release>.
+        displayName: CSI Resizer
+        path: containerImageOverrides.csiResizer
+      - description: A combination of registry and image (e.g. registry.k8s.io/csi-provisioner
+          or ghcr.io/thinkparq/beegfs-csi-driver).
+        displayName: Image
+        path: containerImageOverrides.csiResizer.image
+      - description: A tag (e.g. v2.2.2 or latest).
+        displayName: Tag
+        path: containerImageOverrides.csiResizer.tag
       - description: Defaults to registry.k8s.io/sig-storage/livenessprobe:<the most
           current version at operator release>.
         displayName: Liveness Probe

operator/config/rbac/role.yaml

@@ -84,8 +84,15 @@ rules:
   verbs:
   - get
   - list
+  - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims/status
+  verbs:
+  - patch
 - apiGroups:
   - ""
   resources:
@@ -95,6 +102,15 @@ rules:
   - delete
   - get
   - list
+  - patch
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
   - watch
 - apiGroups:
   - ""

operator/config/samples/beegfs_v1_beegfsdriver.yaml

@@ -40,6 +40,9 @@ spec:
   #     tag:
   #   csiProvisioner:
   #     image:
+  #     tag:
+    #   csiResizer:
+  #     image:
   #     tag:
   #   livenessProbe:
   #     image:

operator/controllers/beegfsdriver_controller.go

@@ -71,12 +71,14 @@ type BeegfsDriverReconciler struct {
 //+kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,resourceNames=privileged,verbs=use
 
 // The operator must have the following permissions in order to grant them to the driver.
-//+kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;create;delete
-//+kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;update
+//+kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;create;delete;patch
+//+kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;update;patch
 //+kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch
 //+kubebuilder:rbac:groups=core,resources=events,verbs=list;watch;create;update;patch
 //+kubebuilder:rbac:groups=storage.k8s.io,resources=csinodes,verbs=get;list;watch
 //+kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch
+//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
+//+kubebuilder:rbac:groups=core,resources=persistentvolumeclaims/status,verbs=patch
 
 // Reconcile is part of the main Kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
@@ -598,6 +600,7 @@ func setImages(log logr.Logger, containers []corev1.Container, overrides beegfsv
 		deploy.ContainerNameBeegfsCsiDriver:        overrides.BeegfsCsiDriver,
 		deploy.ContainerNameCsiNodeDriverRegistrar: overrides.CsiNodeDriverRegistrar,
 		deploy.ContainerNameCsiProvisioner:         overrides.CsiProvisioner,
+		deploy.ContainerNameCsiResizer:             overrides.CsiResizer,
 		deploy.ContainerNameLivenessProbe:          overrides.LivenessProbe,
 	}
 

operator/controllers/beegfsdriver_controller_test.go

@@ -370,6 +370,7 @@ var _ = Describe("Unit tests of helper functions", func() {
 				{Name: deploy.ContainerNameBeegfsCsiDriver, Image: "default.domain/default-driver-image:default-driver-tag"},
 				{Name: deploy.ContainerNameLivenessProbe, Image: "default.domain/default-liveness-image:default-liveness-tag"},
 				{Name: deploy.ContainerNameCsiNodeDriverRegistrar, Image: "default.domain/default-registrar-image:default-registrar-tag"},
+				{Name: deploy.ContainerNameCsiResizer, Image: "default.domain/default-resizer-image:default-resizer-tag"},
 			}
 		})
 
@@ -380,11 +381,13 @@ var _ = Describe("Unit tests of helper functions", func() {
 					BeegfsCsiDriver:        beegfsv1.ContainerImageOverride{Image: "override.domain/override-driver", Tag: "override-tag"},
 					CsiNodeDriverRegistrar: beegfsv1.ContainerImageOverride{Image: "override.domain/override-registrar", Tag: "override-tag"},
 					CsiProvisioner:         beegfsv1.ContainerImageOverride{Image: "override.domain/override-provisioner", Tag: "override-tag"},
+					CsiResizer:             beegfsv1.ContainerImageOverride{Image: "override.domain/override-resizer", Tag: "override-tag"},
 					LivenessProbe:          beegfsv1.ContainerImageOverride{Image: "override.domain/override-liveness", Tag: "override-tag"},
 				}
 				setImages(ctrl.Log, containers, overrides)
 				Expect(getContainerImageForName(deploy.ContainerNameCsiNodeDriverRegistrar, containers)).To(Equal("override.domain/override-registrar:override-tag"))
 				Expect(getContainerImageForName(deploy.ContainerNameCsiProvisioner, containers)).To(Equal("override.domain/override-provisioner:override-tag"))
+				Expect(getContainerImageForName(deploy.ContainerNameCsiResizer, containers)).To(Equal("override.domain/override-resizer:override-tag"))
 				Expect(getContainerImageForName(deploy.ContainerNameBeegfsCsiDriver, containers)).To(Equal("override.domain/override-driver:override-tag"))
 				Expect(getContainerImageForName(deploy.ContainerNameLivenessProbe, containers)).To(Equal("override.domain/override-liveness:override-tag"))
 			})

pkg/beegfs/controllerserver.go

@@ -43,6 +43,7 @@ var (
 	// controllerCaps represents the capabilities of the controller service
 	controllerCaps = []csi.ControllerServiceCapability_RPC_Type{
 		csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
+		csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
 	}
 )
 
@@ -378,7 +379,17 @@ func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnap
 }
 
 func (cs *controllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
-	return nil, status.Error(codes.Unimplemented, "")
+	volumeID := req.GetVolumeId()
+	if len(volumeID) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "Volume ID not provided")
+	}
+	// While currently volume "capacity" has no meaning as far as the driver is concerned, some
+	// applications rely on the capacity of the PV/PVC in the K8s API to make certain decisions.
+	// For these applications it is helpful to support volume resizing.
+	return &csi.ControllerExpandVolumeResponse{
+		CapacityBytes:         req.CapacityRange.RequiredBytes,
+		NodeExpansionRequired: false,
+	}, nil
 }
 
 func (cs *controllerServer) ControllerGetVolume(ctx context.Context, in *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) {