ci(CSI-213): add NFS sanity and bump csi-sanity to 5.3.1

weka · Sep 27, 2024 · eedd7e7 · eedd7e7
1 parent cb63780
commit eedd7e7
Show file tree

Hide file tree

Showing 8 changed files with 204 additions and 78 deletions.
diff --git a/.github/workflows/sanity.yaml b/.github/workflows/sanity.yaml
@@ -54,24 +54,54 @@ jobs:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
       - run: docker-compose -f tests/csi-sanity/docker-compose-snapshotcaps.yaml up $COMPOSE_DEFAULTS
         env:
-          SANITY_FUNCTION: directory_volume_and_snapshots 
+          SANITY_FUNCTION: directory_volume_and_snapshots
 
-  snaphot_volumes_with_2nd_level_shapshots:
+  directory_volume_and_snapshots_nfs:
     if: success() || failure()
     needs: directory_volume_and_snapshots
     runs-on: self-hosted
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+      - run: docker-compose -f tests/csi-sanity/docker-compose-nfs-snapshotcaps.yaml up $COMPOSE_DEFAULTS
+        env:
+          SANITY_FUNCTION: directory_volume_and_snapshots_nfs
+
+  snaphot_volumes_with_2nd_level_shapshots:
+    if: success() || failure()
+    needs: directory_volume_and_snapshots_nfs
+    runs-on: self-hosted
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
       - run: docker-compose -f tests/csi-sanity/docker-compose-snapshotcaps.yaml up $COMPOSE_DEFAULTS
         env:
           SANITY_FUNCTION: snaphot_volumes_with_2nd_level_shapshots  
 
+  snaphot_volumes_with_2nd_level_shapshots_nfs:
+      if: success() || failure()
+      needs: snaphot_volumes_with_2nd_level_shapshots
+      runs-on: self-hosted
+      steps:
+        - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+        - run: docker-compose -f tests/csi-sanity/docker-compose-nfs-snapshotcaps.yaml up $COMPOSE_DEFAULTS
+          env:
+            SANITY_FUNCTION: snaphot_volumes_with_2nd_level_shapshots_nfs
+
   filesystem_volumes:
     if: success() || failure()
-    needs: snaphot_volumes_with_2nd_level_shapshots
+    needs: snaphot_volumes_with_2nd_level_shapshots_nfs
     runs-on: self-hosted
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
       - run: docker-compose -f tests/csi-sanity/docker-compose-snapshotcaps.yaml up $COMPOSE_DEFAULTS
         env:
           SANITY_FUNCTION: filesystem_volumes 
+
+  filesystem_volumes_nfs:
+    if: success() || failure()
+    needs: filesystem_volumes
+    runs-on: self-hosted
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+      - run: docker-compose -f tests/csi-sanity/docker-compose-nfs-snapshotcaps.yaml up $COMPOSE_DEFAULTS
+        env:
+          SANITY_FUNCTION: filesystem_volumes_nfs
diff --git a/pkg/wekafs/apiclient/filesystem.go b/pkg/wekafs/apiclient/filesystem.go
@@ -222,6 +222,9 @@ func (a *ApiClient) EnsureNoNfsPermissionsForFilesystem(ctx context.Context, fsN
 	if err != nil {
 		logger.Error().Err(err).Str("filesystem", fsName).Msg("Failed to list NFS permissions")
 	}
+	if len(*permissions) > 0 {
+		logger.Debug().Int("permissions", len(*permissions)).Str("filesystem", fsName).Msg("Found stale NFS permissions, deleting")
+	}
 	for _, p := range *permissions {
 		err = a.DeleteNfsPermission(ctx, &NfsPermissionDeleteRequest{Uid: p.Uid})
 		if err != nil {

diff --git a/tests/csi-sanity/_docker_run_sanity.sh b/tests/csi-sanity/_docker_run_sanity.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-rm -rf /tmp/weka-csi-test/sanity-workspace/
+rm -rf /tmp/weka-csi-test/sanity-workspace
 rm -rf /tmp/weka-csi-test/filesystems
 rm -rf /tmp/csi-test-staging
 
@@ -13,7 +13,7 @@ if \
   csi-sanity -csi.stagingdir /tmp/csi-test-staging \
     --csi.controllerendpoint /tmp/weka-csi-test/controller-no-snaps.sock \
     --csi.endpoint /tmp/weka-csi-test/node-no-snaps.sock \
-    -csi.mountdir=/tmp/weka-csi-test/sanity-workspace/ \
+    -csi.mountdir=/tmp/weka-csi-test/sanity-workspace \
     -ginkgo.reportPassed \
     -ginkgo.failFast \
     -ginkgo.progress \
@@ -37,7 +37,7 @@ if \
     --csi.controllerendpoint /tmp/weka-csi-test/controller-no-snaps.sock \
     --csi.endpoint /tmp/weka-csi-test/node-no-snaps.sock \
     -csi.secrets /test/wekafs-api-secret.yaml \
-    -csi.mountdir=/tmp/weka-csi-test/sanity-workspace/ \
+    -csi.mountdir=/tmp/weka-csi-test/sanity-workspace \
     -ginkgo.seed 0 \
     -ginkgo.failFast \
     -ginkgo.progress \
@@ -60,7 +60,7 @@ if \
     --csi.controllerendpoint /tmp/weka-csi-test/controller-no-snaps.sock \
     --csi.endpoint /tmp/weka-csi-test/node-no-snaps.sock \
     -csi.secrets /test/wekafs-api-secret.yaml \
-    -csi.mountdir=/tmp/weka-csi-test/sanity-workspace/ \
+    -csi.mountdir=/tmp/weka-csi-test/sanity-workspace \
     -csi.testvolumeparameters /test/wekafs-fs.yaml \
     -ginkgo.seed 0 \
     -ginkgo.failFast \
@@ -82,7 +82,7 @@ if \
     --csi.controllerendpoint /tmp/weka-csi-test/controller.sock \
     --csi.endpoint /tmp/weka-csi-test/node.sock \
     -csi.secrets /test/wekafs-api-secret.yaml \
-    -csi.mountdir=/tmp/weka-csi-test/sanity-workspace/ \
+    -csi.mountdir=/tmp/weka-csi-test/sanity-workspace \
     -ginkgo.seed 0 \
     -ginkgo.failFast \
     -ginkgo.progress \
@@ -105,7 +105,7 @@ if \
     --csi.controllerendpoint /tmp/weka-csi-test/controller.sock \
     --csi.endpoint /tmp/weka-csi-test/node.sock \
     -csi.secrets /test/wekafs-api-secret.yaml \
-    -csi.mountdir=/tmp/weka-csi-test/sanity-workspace/ \
+    -csi.mountdir=/tmp/weka-csi-test/sanity-workspace \
     -ginkgo.seed 0 \
     -ginkgo.failFast \
     -ginkgo.progress \
@@ -127,7 +127,7 @@ if \
     --csi.controllerendpoint /tmp/weka-csi-test/controller.sock \
     --csi.endpoint /tmp/weka-csi-test/node.sock \
     -csi.secrets /test/wekafs-api-secret.yaml \
-    -csi.mountdir=/tmp/weka-csi-test/sanity-workspace/ \
+    -csi.mountdir=/tmp/weka-csi-test/sanity-workspace \
     -ginkgo.seed 0 \
     -ginkgo.failFast \
     -ginkgo.progress \

diff --git a/tests/csi-sanity/docker-compose-nfs-snapshotcaps.yaml b/tests/csi-sanity/docker-compose-nfs-snapshotcaps.yaml
@@ -0,0 +1,40 @@
+version: '3.8'
+
+services:
+  plugin_controller:
+    image: sanity
+    command: wekafsplugin -nodeid 1 -v 6 --allowinsecurehttps --endpoint=unix://tmp/weka-csi-test/controller.sock -metricsport=9091  --csimode=controller -enablemetrics -allowautofscreation -allowautofsexpansion -allowsnapshotsoflegacyvolumes -alwaysallowsnapshotvolumes -usenfs -interfacegroupname=NFS
+    volumes:
+      - test-volume:/tmp/weka-csi-test
+    privileged: true
+    network_mode: host
+    healthcheck:
+      test: test -f /tmp/weka-csi-test/controller.sock
+      start_period: 1s
+      timeout: 1s
+      retries: 10
+      interval: 3s
+  plugin_node:
+    image: sanity
+    command: wekafsplugin -nodeid 1 -v 6 --allowinsecurehttps --endpoint=unix://tmp/weka-csi-test/node.sock -metricsport=9092 --csimode=node -enablemetrics -allowautofscreation -allowautofsexpansion -allowsnapshotsoflegacyvolumes -alwaysallowsnapshotvolumes -usenfs -interfacegroupname=NFS
+    volumes:
+      - test-volume:/tmp/weka-csi-test
+    privileged: true
+    network_mode: host
+    healthcheck:
+      test: test -f /tmp/weka-csi-test/node.sock
+      start_period: 1s
+      timeout: 1s
+      retries: 10
+      interval: 3s
+  sanity:
+    image: sanity
+    command: run_sanity ${SANITY_FUNCTION} ${SANITY_VERBOSITY}
+    network_mode: host
+    volumes:
+      - test-volume:/tmp/weka-csi-test
+    depends_on:
+      - plugin_controller
+      - plugin_node
+volumes:
+  test-volume:
diff --git a/tests/csi-sanity/docker-compose.yaml b/tests/csi-sanity/docker-compose.yaml
@@ -33,6 +33,22 @@ services:
     privileged: true
     ports:
       - "9004:9090"
+  plugin_controller_nfs:
+    build: .
+    command: wekafsplugin -nodeid 1 -v 9 --allowinsecurehttps --endpoint=unix://tmp/weka-csi-test/controller-nfs.sock --debugpath=/tmp/weka-csi-test/filesystems --csimode=controller -enablemetrics -allowautofscreation -allowautofsexpansion -allowsnapshotsoflegacyvolumes -alwaysallowsnapshotvolumes -usenfs
+    volumes:
+      - test-volume:/tmp/weka-csi-test
+    privileged: true
+    ports:
+      - "9005:9090"
+  plugin_node_nfs:
+    build: .
+    command: wekafsplugin -nodeid 1 -v 9 --allowinsecurehttps --endpoint=unix://tmp/weka-csi-test/node.sock --debugpath=/tmp/weka-csi-test/filesystems --csimode=node -enablemetrics -allowautofscreation -allowautofsexpansion -allowsnapshotsoflegacyvolumes  -alwaysallowsnapshotvolumes -usenfs
+    volumes:
+      - test-volume:/tmp/weka-csi-test
+    privileged: true
+    ports:
+      - "9006:9090"
   sanity:
     build: .
     command: run_sanity

diff --git a/tests/csi-sanity/ga-Dockerfile b/tests/csi-sanity/ga-Dockerfile
@@ -1,25 +1,25 @@
-FROM golang:1.22-alpine
+FROM golang:1.22-alpine AS builder
 
 WORKDIR /app
 
 COPY go.mod .
 COPY go.sum .
 RUN go mod download
 
-RUN go install -v github.com/kubernetes-csi/csi-test/v5/cmd/csi-sanity@v5.0.0
+RUN go install -v github.com/kubernetes-csi/csi-test/v5/cmd/csi-sanity@v5.3.1
 
 COPY cmd cmd
 COPY pkg pkg
 
 RUN CGO_ENABLED=0 GOOS="linux" GOARCH="amd64" go build -a -ldflags '-X main.version='$VERSION' -extldflags "-static"' -o wekafsplugin ./cmd/wekafsplugin/main.go
 
 
-FROM golang:1.21-alpine
+FROM golang:1.22-alpine
 
 ADD tests/csi-sanity/wekafs-dirv1.yaml /test/wekafs-dirv1.yaml
 ADD tests/csi-sanity/wekafs-fs.yaml /test/wekafs-fs.yaml
 ADD tests/csi-sanity/wekafs-snapvol.yaml /test/wekafs-snapvol.yaml
 ADD tests/csi-sanity/wekafs-api-secret.yaml /test/wekafs-api-secret.yaml
 ADD tests/csi-sanity/ga_docker_run_sanity.sh /usr/bin/run_sanity
-COPY --from=0 /go/bin/csi-sanity /usr/local/bin/csi-sanity
-COPY --from=0 /app/wekafsplugin  /usr/local/bin/wekafsplugin
+COPY --from=builder /go/bin/csi-sanity /usr/local/bin/csi-sanity
+COPY --from=builder /app/wekafsplugin  /usr/local/bin/wekafsplugin