diff --git a/.github/release.yaml b/.github/release.yaml index 62e642bef..d6183e671 100644 --- a/.github/release.yaml +++ b/.github/release.yaml @@ -10,8 +10,10 @@ changelog: - title: Bug Fixes labels: - fix + - field issue - title: Miscellaneous labels: - dependencies - ci - documentation + - sidecar diff --git a/.github/workflows/sanity.yaml b/.github/workflows/sanity.yaml index 4cdc02f88..047098140 100644 --- a/.github/workflows/sanity.yaml +++ b/.github/workflows/sanity.yaml @@ -23,12 +23,17 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - run: echo "${{ secrets.WEKAFS_API_SECRET_YAML }}" > tests/csi-sanity/wekafs-api-secret.yaml - - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} - uses: docker/build-push-action@v6 with: context: . file: tests/csi-sanity/ga-Dockerfile + cache-from: type=gha + cache-to: type=gha,mode=max tags: sanity:latest load: true @@ -41,29 +46,9 @@ jobs: env: SANITY_FUNCTION: legacy_sanity - directory_volume_no_snapshots: - if: success() || failure() # always() can't be canceled - needs: legacy_sanity - runs-on: self-hosted - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - run: docker-compose -f tests/csi-sanity/docker-compose-nosnapshotcaps.yaml up $COMPOSE_DEFAULTS - env: - SANITY_FUNCTION: directory_volume_no_snapshots - - fs_volume_no_snapshots: - if: success() || failure() - needs: directory_volume_no_snapshots - runs-on: self-hosted - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - run: docker-compose -f tests/csi-sanity/docker-compose-nosnapshotcaps.yaml up $COMPOSE_DEFAULTS - env: - SANITY_FUNCTION: fs_volume_no_snapshots - directory_volume_and_snapshots: if: success() || failure() - needs: fs_volume_no_snapshots + needs: legacy_sanity runs-on: self-hosted steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 diff --git a/Dockerfile b/Dockerfile index 1391ff267..7fb187e79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,6 @@ -FROM golang:1.22-alpine as go-builder +FROM golang:1.22-alpine AS go-builder # https://stackoverflow.com/questions/36279253/go-compiled-binary-wont-run-in-an-alpine-docker-container-on-ubuntu-host -RUN apk add --no-cache libc6-compat gcc -RUN apk add musl-dev +RUN apk add --no-cache libc6-compat gcc musl-dev COPY go.mod /src/go.mod COPY go.sum /src/go.sum WORKDIR /src @@ -22,17 +21,20 @@ RUN true RUN echo Building package RUN CGO_ENABLED=0 GOOS="linux" GOARCH="amd64" go build -a -ldflags '-X main.version='$VERSION' -extldflags "-static"' -o "/bin/wekafsplugin" /src/cmd/* +FROM registry.k8s.io/kubernetes/kubectl:v1.31.1 AS kubectl FROM alpine:3.18 LABEL maintainers="WekaIO, LTD" LABEL description="Weka CSI Driver" -# Add util-linux to get a new version of losetup. -RUN apk add util-linux libselinux libselinux-utils util-linux pciutils usbutils coreutils binutils findutils grep bash + +ADD --chmod=777 https://github.com/tigrawap/locar/releases/download/0.4.0/locar_linux_amd64 /locar +RUN apk add --no-cache util-linux libselinux libselinux-utils util-linux \ + pciutils usbutils coreutils binutils findutils \ + grep bash nfs-utils rpcbind ca-certificates jq # Update CA certificates -RUN apk add ca-certificates RUN update-ca-certificates -ADD https://github.com/tigrawap/locar/releases/download/0.4.0/locar_linux_amd64 /locar -RUN chmod +x /locar +COPY --from=kubectl /bin/kubectl /bin/kubectl COPY --from=go-builder /bin/wekafsplugin /wekafsplugin ARG binary=/bin/wekafsplugin +EXPOSE 2049 111/tcp 111/udp ENTRYPOINT ["/wekafsplugin"] diff --git a/README.md b/README.md index a5fa965fc..3ad6adf02 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # CSI WekaFS Driver Helm chart for Deployment of WekaIO Container Storage Interface (CSI) plugin for WekaFS - the world fastest filesystem -![Version: 2.4.1](https://img.shields.io/badge/Version-2.4.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.4.1](https://img.shields.io/badge/AppVersion-v2.4.1-informational?style=flat-square) +![Version: 2.4.2-SNAPSHOT.99.90161ea](https://img.shields.io/badge/Version-2.4.2--SNAPSHOT.99.90161ea-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.4.2-SNAPSHOT.99.90161ea](https://img.shields.io/badge/AppVersion-v2.4.2--SNAPSHOT.99.90161ea-informational?style=flat-square) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/csi-wekafs)](https://artifacthub.io/packages/search?repo=csi-wekafs) @@ -15,7 +15,7 @@ https://github.com/weka/csi-wekafs | WekaIO, Inc. | | | ## Pre-requisite -- Kubernetes cluster of version 1.18 or later. Although older versions from 1.13 and up should work, they were not tested +- Kubernetes cluster of version 1.20 or later is recommended. Minimum version is 1.17 - Access to terminal with `kubectl` installed - Weka system pre-configured and Weka client installed and registered in cluster for each Kubernetes node @@ -26,9 +26,10 @@ https://github.com/weka/csi-wekafs ## Usage - [Deploy an Example application](docs/usage.md) - [SELinux Support & Installation Notes](selinux/README.md) +- [Using Weka CSI Plugin with NFS transport](docs/NFS.md) ## Additional Documentation -- [Official Weka CSI Plugin documentation](https://docs.weka.io/appendix/weka-csi-plugin) +- [Official Weka CSI Plugin documentation](https://docs.weka.io/appendices/weka-csi-plugin) ## Building the binaries If you want to build the driver yourself, you can do so with the following command from the root directory: @@ -43,16 +44,17 @@ make build |-----|------|---------|-------------| | dynamicProvisionPath | string | `"csi-volumes"` | Directory in root of file system where dynamic volumes are provisioned | | csiDriverName | string | `"csi.weka.io"` | Name of the driver (and provisioner) | -| csiDriverVersion | string | `"2.4.1"` | CSI driver version | -| images.livenessprobesidecar | string | `"registry.k8s.io/sig-storage/livenessprobe:v2.12.0"` | CSI liveness probe sidecar image URL | -| images.attachersidecar | string | `"registry.k8s.io/sig-storage/csi-attacher:v4.5.0"` | CSI attacher sidecar image URL | -| images.provisionersidecar | string | `"registry.k8s.io/sig-storage/csi-provisioner:v4.0.0"` | CSI provisioner sidecar image URL | -| images.registrarsidecar | string | `"registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0"` | CSI registrar sidercar | -| images.resizersidecar | string | `"registry.k8s.io/sig-storage/csi-resizer:v1.9.3"` | CSI resizer sidecar image URL | -| images.snapshottersidecar | string | `"registry.k8s.io/sig-storage/csi-snapshotter:v6.3.3"` | CSI snapshotter sidecar image URL | -| images.nodeinfo | string | `"quay.io/weka.io/kubectl-sidecar:v1.29.2-1"` | CSI nodeinfo sidecar image URL, used for reading node metadata | +| csiDriverVersion | string | `"2.4.2-SNAPSHOT.99.90161ea"` | CSI driver version | +| images.livenessprobesidecar | string | `"registry.k8s.io/sig-storage/livenessprobe:v2.14.0"` | CSI liveness probe sidecar image URL | +| images.attachersidecar | string | `"registry.k8s.io/sig-storage/csi-attacher:v4.7.0"` | CSI attacher sidecar image URL | +| images.provisionersidecar | string | `"registry.k8s.io/sig-storage/csi-provisioner:v5.1.0"` | CSI provisioner sidecar image URL | +| images.registrarsidecar | string | `"registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.12.0"` | CSI registrar sidercar | +| images.resizersidecar | string | `"registry.k8s.io/sig-storage/csi-resizer:v1.12.0"` | CSI resizer sidecar image URL | +| images.snapshottersidecar | string | `"registry.k8s.io/sig-storage/csi-snapshotter:v8.1.0"` | CSI snapshotter sidecar image URL | +| images.nodeinfo | string | `"quay.io/weka.io/csi-wekafs"` | CSI nodeinfo sidecar image URL, used for reading node metadata | | images.csidriver | string | `"quay.io/weka.io/csi-wekafs"` | CSI driver main image URL | -| images.csidriverTag | string | `"2.4.1"` | CSI driver tag | +| images.csidriverTag | string | `"2.4.2-SNAPSHOT.99.90161ea"` | CSI driver tag | +| imagePullSecret | string | `""` | image pull secret required for image download. Must have permissions to access all images above. Should be used in case of private registry that requires authentication | | globalPluginTolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"}]` | Tolerations for all CSI driver components | | controllerPluginTolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"}]` | Tolerations for CSI controller component only (by default same as global) | | nodePluginTolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"}]` | Tolerations for CSI node component only (by default same as global) | @@ -76,11 +78,12 @@ make build | selinuxNodeLabel | string | `"csi.weka.io/selinux_enabled"` | This label must be set to `"true"` on SELinux-enabled Kubernetes nodes, e.g., to run the node server in secure mode on SELinux-enabled node, the node must have label `csi.weka.io/selinux_enabled="true"` | | kubeletPath | string | `"/var/lib/kubelet"` | kubelet path, in cases Kubernetes is installed not in default folder | | metrics.enabled | bool | `true` | Enable Prometheus Metrics | -| metrics.port | int | `9090` | Metrics port | +| metrics.controllerPort | int | `9090` | Metrics port for Controller Server | | metrics.provisionerPort | int | `9091` | Provisioner metrics port | | metrics.resizerPort | int | `9092` | Resizer metrics port | | metrics.snapshotterPort | int | `9093` | Snapshotter metrics port | -| hostNetwork | bool | `false` | Set to true to use host networking | +| metrics.nodePort | int | `9094` | Metrics port for Node Serer | +| hostNetwork | bool | `false` | Set to true to use host networking. Will be always set to true when using NFS mount protocol | | pluginConfig.fsGroupPolicy | string | `"File"` | WARNING: Changing this value might require uninstall and re-install of the plugin | | pluginConfig.allowInsecureHttps | bool | `false` | Allow insecure HTTPS (skip TLS certificate verification) | | pluginConfig.objectNaming.volumePrefix | string | `"csivol-"` | Prefix that will be added to names of Weka cluster filesystems / snapshots assocciated with CSI volume, must not exceed 7 symbols. | @@ -91,6 +94,12 @@ make build | pluginConfig.allowedOperations.snapshotDirectoryVolumes | bool | `false` | Create snapshots of legacy (dir/v1) volumes. By default disabled. Note: when enabled, for every legacy volume snapshot, a full filesystem snapshot will be created (wasteful) | | pluginConfig.allowedOperations.snapshotVolumesWithoutQuotaEnforcement | bool | `false` | Allow creation of snapshot-backed volumes even on unsupported Weka cluster versions, off by default Note: On versions of Weka < v4.2 snapshot-backed volume capacity cannot be enforced | | pluginConfig.mutuallyExclusiveMountOptions[0] | string | `"readcache,writecache,coherent,forcedirect"` | | +| pluginConfig.mutuallyExclusiveMountOptions[1] | string | `"sync,async"` | | +| pluginConfig.mountProtocol.useNfs | bool | `false` | Use NFS transport for mounting Weka filesystems, off by default | +| pluginConfig.mountProtocol.allowNfsFailback | bool | `false` | Allow Failback to NFS transport if Weka client fails to mount filesystem using native protocol | +| pluginConfig.mountProtocol.interfaceGroupName | string | `""` | Specify name of NFS interface group to use for mounting Weka filesystems. If not set, first NFS interface group will be used | +| pluginConfig.mountProtocol.clientGroupName | string | `""` | Specify existing client group name for NFS configuration. If not set, "WekaCSIPluginClients" group will be created | +| pluginConfig.mountProtocol.nfsProtocolVersion | string | `"4.1"` | Specify NFS protocol version to use for mounting Weka filesystems. Default is "4.1", consult Weka documentation for supported versions | ---------------------------------------------- Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/README.md.gotmpl b/README.md.gotmpl index 7b7d6e559..bb7e87288 100644 --- a/README.md.gotmpl +++ b/README.md.gotmpl @@ -11,7 +11,7 @@ {{ template "chart.maintainersSection" . }} ## Pre-requisite -- Kubernetes cluster of version 1.18 or later. Although older versions from 1.13 and up should work, they were not tested +- Kubernetes cluster of version 1.20 or later is recommended. Minimum version is 1.17 - Access to terminal with `kubectl` installed - Weka system pre-configured and Weka client installed and registered in cluster for each Kubernetes node @@ -22,9 +22,10 @@ ## Usage - [Deploy an Example application](docs/usage.md) - [SELinux Support & Installation Notes](selinux/README.md) +- [Using Weka CSI Plugin with NFS transport](docs/NFS.md) ## Additional Documentation -- [Official Weka CSI Plugin documentation](https://docs.weka.io/appendix/weka-csi-plugin) +- [Official Weka CSI Plugin documentation](https://docs.weka.io/appendices/weka-csi-plugin) ## Building the binaries If you want to build the driver yourself, you can do so with the following command from the root directory: diff --git a/charts/csi-wekafsplugin/Chart.yaml b/charts/csi-wekafsplugin/Chart.yaml index b9944de0e..3d006947d 100644 --- a/charts/csi-wekafsplugin/Chart.yaml +++ b/charts/csi-wekafsplugin/Chart.yaml @@ -6,12 +6,12 @@ maintainers: email: csi@weka.io url: https://weka.io sources: - - https://github.com/weka/csi-wekafs/tree/v2.4.1 + - https://github.com/weka/csi-wekafs/tree/v$CHART_VERSION/charts/csi-wekafsplugin home: https://github.com/weka/csi-wekafs icon: https://weka.github.io/csi-wekafs/logo.png type: application -version: 2.4.1 -appVersion: v2.4.1 +version: 2.4.2-SNAPSHOT.99.90161ea +appVersion: v2.4.2-SNAPSHOT.99.90161ea keywords: [storage, filesystem, HPC] annotations: artifacthub.io/category: "storage" diff --git a/charts/csi-wekafsplugin/README.md b/charts/csi-wekafsplugin/README.md index 43a6a01d1..c37c96e2c 100644 --- a/charts/csi-wekafsplugin/README.md +++ b/charts/csi-wekafsplugin/README.md @@ -3,7 +3,7 @@ Helm chart for Deployment of WekaIO Container Storage Interface (CSI) plugin for [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/csi-wekafs)](https://artifacthub.io/packages/search?repo=csi-wekafs) -![Version: 2.4.1](https://img.shields.io/badge/Version-2.4.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.4.1](https://img.shields.io/badge/AppVersion-v2.4.1-informational?style=flat-square) +![Version: 2.4.2-SNAPSHOT.99.90161ea](https://img.shields.io/badge/Version-2.4.2--SNAPSHOT.99.90161ea-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.4.2-SNAPSHOT.99.90161ea](https://img.shields.io/badge/AppVersion-v2.4.2--SNAPSHOT.99.90161ea-informational?style=flat-square) ## Homepage https://github.com/weka/csi-wekafs @@ -37,14 +37,14 @@ helm install csi-wekafsplugin csi-wekafs/csi-wekafsplugin --namespace csi-wekafs > However, for sake of more convenient migration, a `legacySecretName` parameter can be set that will > bind existing legacy volumes to a Weka cluster API and allow volume expansion. > -> For further information, refer [Official Weka CSI Plugin documentation](https://docs.weka.io/appendix/weka-csi-plugin) +> For further information, refer [Official Weka CSI Plugin documentation](https://docs.weka.io/appendices/weka-csi-plugin) ## Usage - [Deploy an Example application](https://github.com/weka/csi-wekafs/blob/master/docs/usage.md) - [SELinux Support & Installation Notes](https://github.com/weka/csi-wekafs/blob/master/selinux/README.md) ## Additional Documentation -- [Official Weka CSI Plugin documentation](https://docs.weka.io/appendix/weka-csi-plugin) +- [Official Weka CSI Plugin documentation](https://docs.weka.io/appendices/weka-csi-plugin) ## Values @@ -52,16 +52,17 @@ helm install csi-wekafsplugin csi-wekafs/csi-wekafsplugin --namespace csi-wekafs |-----|------|---------|-------------| | dynamicProvisionPath | string | `"csi-volumes"` | Directory in root of file system where dynamic volumes are provisioned | | csiDriverName | string | `"csi.weka.io"` | Name of the driver (and provisioner) | -| csiDriverVersion | string | `"2.4.1"` | CSI driver version | -| images.livenessprobesidecar | string | `"registry.k8s.io/sig-storage/livenessprobe:v2.12.0"` | CSI liveness probe sidecar image URL | -| images.attachersidecar | string | `"registry.k8s.io/sig-storage/csi-attacher:v4.5.0"` | CSI attacher sidecar image URL | -| images.provisionersidecar | string | `"registry.k8s.io/sig-storage/csi-provisioner:v4.0.0"` | CSI provisioner sidecar image URL | -| images.registrarsidecar | string | `"registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0"` | CSI registrar sidercar | -| images.resizersidecar | string | `"registry.k8s.io/sig-storage/csi-resizer:v1.9.3"` | CSI resizer sidecar image URL | -| images.snapshottersidecar | string | `"registry.k8s.io/sig-storage/csi-snapshotter:v6.3.3"` | CSI snapshotter sidecar image URL | -| images.nodeinfo | string | `"quay.io/weka.io/kubectl-sidecar:v1.29.2-1"` | CSI nodeinfo sidecar image URL, used for reading node metadata | +| csiDriverVersion | string | `"2.4.2-SNAPSHOT.99.90161ea"` | CSI driver version | +| images.livenessprobesidecar | string | `"registry.k8s.io/sig-storage/livenessprobe:v2.14.0"` | CSI liveness probe sidecar image URL | +| images.attachersidecar | string | `"registry.k8s.io/sig-storage/csi-attacher:v4.7.0"` | CSI attacher sidecar image URL | +| images.provisionersidecar | string | `"registry.k8s.io/sig-storage/csi-provisioner:v5.1.0"` | CSI provisioner sidecar image URL | +| images.registrarsidecar | string | `"registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.12.0"` | CSI registrar sidercar | +| images.resizersidecar | string | `"registry.k8s.io/sig-storage/csi-resizer:v1.12.0"` | CSI resizer sidecar image URL | +| images.snapshottersidecar | string | `"registry.k8s.io/sig-storage/csi-snapshotter:v8.1.0"` | CSI snapshotter sidecar image URL | +| images.nodeinfo | string | `"quay.io/weka.io/csi-wekafs"` | CSI nodeinfo sidecar image URL, used for reading node metadata | | images.csidriver | string | `"quay.io/weka.io/csi-wekafs"` | CSI driver main image URL | -| images.csidriverTag | string | `"2.4.1"` | CSI driver tag | +| images.csidriverTag | string | `"2.4.2-SNAPSHOT.99.90161ea"` | CSI driver tag | +| imagePullSecret | string | `""` | image pull secret required for image download. Must have permissions to access all images above. Should be used in case of private registry that requires authentication | | globalPluginTolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"}]` | Tolerations for all CSI driver components | | controllerPluginTolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"}]` | Tolerations for CSI controller component only (by default same as global) | | nodePluginTolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"}]` | Tolerations for CSI node component only (by default same as global) | @@ -85,11 +86,12 @@ helm install csi-wekafsplugin csi-wekafs/csi-wekafsplugin --namespace csi-wekafs | selinuxNodeLabel | string | `"csi.weka.io/selinux_enabled"` | This label must be set to `"true"` on SELinux-enabled Kubernetes nodes, e.g., to run the node server in secure mode on SELinux-enabled node, the node must have label `csi.weka.io/selinux_enabled="true"` | | kubeletPath | string | `"/var/lib/kubelet"` | kubelet path, in cases Kubernetes is installed not in default folder | | metrics.enabled | bool | `true` | Enable Prometheus Metrics | -| metrics.port | int | `9090` | Metrics port | +| metrics.controllerPort | int | `9090` | Metrics port for Controller Server | | metrics.provisionerPort | int | `9091` | Provisioner metrics port | | metrics.resizerPort | int | `9092` | Resizer metrics port | | metrics.snapshotterPort | int | `9093` | Snapshotter metrics port | -| hostNetwork | bool | `false` | Set to true to use host networking | +| metrics.nodePort | int | `9094` | Metrics port for Node Serer | +| hostNetwork | bool | `false` | Set to true to use host networking. Will be always set to true when using NFS mount protocol | | pluginConfig.fsGroupPolicy | string | `"File"` | WARNING: Changing this value might require uninstall and re-install of the plugin | | pluginConfig.allowInsecureHttps | bool | `false` | Allow insecure HTTPS (skip TLS certificate verification) | | pluginConfig.objectNaming.volumePrefix | string | `"csivol-"` | Prefix that will be added to names of Weka cluster filesystems / snapshots assocciated with CSI volume, must not exceed 7 symbols. | @@ -100,6 +102,12 @@ helm install csi-wekafsplugin csi-wekafs/csi-wekafsplugin --namespace csi-wekafs | pluginConfig.allowedOperations.snapshotDirectoryVolumes | bool | `false` | Create snapshots of legacy (dir/v1) volumes. By default disabled. Note: when enabled, for every legacy volume snapshot, a full filesystem snapshot will be created (wasteful) | | pluginConfig.allowedOperations.snapshotVolumesWithoutQuotaEnforcement | bool | `false` | Allow creation of snapshot-backed volumes even on unsupported Weka cluster versions, off by default Note: On versions of Weka < v4.2 snapshot-backed volume capacity cannot be enforced | | pluginConfig.mutuallyExclusiveMountOptions[0] | string | `"readcache,writecache,coherent,forcedirect"` | | +| pluginConfig.mutuallyExclusiveMountOptions[1] | string | `"sync,async"` | | +| pluginConfig.mountProtocol.useNfs | bool | `false` | Use NFS transport for mounting Weka filesystems, off by default | +| pluginConfig.mountProtocol.allowNfsFailback | bool | `false` | Allow Failback to NFS transport if Weka client fails to mount filesystem using native protocol | +| pluginConfig.mountProtocol.interfaceGroupName | string | `""` | Specify name of NFS interface group to use for mounting Weka filesystems. If not set, first NFS interface group will be used | +| pluginConfig.mountProtocol.clientGroupName | string | `""` | Specify existing client group name for NFS configuration. If not set, "WekaCSIPluginClients" group will be created | +| pluginConfig.mountProtocol.nfsProtocolVersion | string | `"4.1"` | Specify NFS protocol version to use for mounting Weka filesystems. Default is "4.1", consult Weka documentation for supported versions | ---------------------------------------------- Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/charts/csi-wekafsplugin/README.md.gotmpl b/charts/csi-wekafsplugin/README.md.gotmpl index b7bcc02ec..1bae930df 100644 --- a/charts/csi-wekafsplugin/README.md.gotmpl +++ b/charts/csi-wekafsplugin/README.md.gotmpl @@ -33,14 +33,14 @@ helm install csi-wekafsplugin csi-wekafs/csi-wekafsplugin --namespace csi-wekafs > However, for sake of more convenient migration, a `legacySecretName` parameter can be set that will > bind existing legacy volumes to a Weka cluster API and allow volume expansion. > -> For further information, refer [Official Weka CSI Plugin documentation](https://docs.weka.io/appendix/weka-csi-plugin) +> For further information, refer [Official Weka CSI Plugin documentation](https://docs.weka.io/appendices/weka-csi-plugin) ## Usage - [Deploy an Example application](https://github.com/weka/csi-wekafs/blob/master/docs/usage.md) - [SELinux Support & Installation Notes](https://github.com/weka/csi-wekafs/blob/master/selinux/README.md) ## Additional Documentation -- [Official Weka CSI Plugin documentation](https://docs.weka.io/appendix/weka-csi-plugin) +- [Official Weka CSI Plugin documentation](https://docs.weka.io/appendices/weka-csi-plugin) {{ template "chart.requirementsSection" . }} diff --git a/charts/csi-wekafsplugin/templates/NOTES.txt b/charts/csi-wekafsplugin/templates/NOTES.txt index 7bf60d2f3..c1a798107 100644 --- a/charts/csi-wekafsplugin/templates/NOTES.txt +++ b/charts/csi-wekafsplugin/templates/NOTES.txt @@ -8,7 +8,7 @@ To learn more about the release, try: $ helm status -n {{ .Release.Namespace}} {{ .Release.Name }} $ helm get all -n {{ .Release.Namespace}} {{ .Release.Name }} -Official Weka CSI Plugin documentation can be found here: https://docs.weka.io/appendix/weka-csi-plugin +Official Weka CSI Plugin documentation can be found here: https://docs.weka.io/appendices/weka-csi-plugin Examples on how to configure a storage class and start using the driver are here: https://github.com/weka/csi-wekafs/tree/master/examples @@ -25,3 +25,21 @@ https://github.com/weka/csi-wekafs/tree/master/examples | NEW FEATURES RELY ON API CONNECTIVITY TO WEKA CLUSTER AND WILL NOT BE SUPPORTED ON API-UNBOUND VOLUMES. | | PLEASE MAKE SURE TO MIGRATE ALL EXISTING VOLUMES TO API-BASED SCHEME PRIOR TO NEXT VERSION UPGRADE. | ------------------------------------------------------------------------------------------------------------ + +{{- if or .Values.pluginConfig.mountProtocol.useNfs .Values.pluginConfig.mountProtocol.allowNfsFailback }} +-------------------------------------------------- WARNING ------------------------------------------------- +{{- if .Values.pluginConfig.mountProtocol.useNfs }} +| WARNING: NFS PROTOCOL IS ENFORCED AND WILL ALWAYS BE USED FOR MOUNTING WEKA FILESYSTEMS! | +| NFS TRANSPORT DOES NOT PROVIDE MAXIMUM PERFORMANCE AND IS NOT RECOMMENDED FOR PRODUCTION USE. | +{{- else }} +| WARNING: NFS MOUNT PROTOCOL FAILBACK IS ENABLED, AND NFS MOUNTS WILL BE USED IF WEKA IS NOT INSTALLED. | +| NFS TRANSPORT DOES NOT PROVIDE MAXIMUM PERFORMANCE AND IS NOT RECOMMENDED FOR PRODUCTION USE. | +| HOWEVER, IN CERTAIN CASES WHEN WEKA CLIENT INSTALLATION IS NOT POSSIBLE, NFS MOUNTS WILL BE USED. | +| IF WEKA CLIENT IS INSTALLED ON NODES AFTER CSI PLUGIN INSTALLATION, RESTART IS REQUIRED FOR THE | +| CORRESPONDENT CSI PLUGIN COMPONENTS RUNNING ON THE NODE TO SWITCH BACK TO WEKAFS PROTOCOL MOUNTING. | +{{- end }} +| MAKE SURE THAT AT LEAST ONE INTERFACE GROUP IS CONFIGURED ON WEKA CLUSTER, OTHERWISE PROVISION WILL FAIL | +| REFER TO THE DOCUMENTATION ABOVE FOR MORE INFORMATION ON NFS INTERFACE GROUP CONFIGURATION. | +| REFER TO WEKA CUSTOMER SUCCESS TEAM FOR RECOMMENDED CONFIGURATION AND BEST PRACTICES | +------------------------------------------------------------------------------------------------------------ +{{- end }} diff --git a/charts/csi-wekafsplugin/templates/controllerserver-security-context-constraint.yaml b/charts/csi-wekafsplugin/templates/controllerserver-security-context-constraint.yaml index c9bcb02d8..8a463f6cb 100644 --- a/charts/csi-wekafsplugin/templates/controllerserver-security-context-constraint.yaml +++ b/charts/csi-wekafsplugin/templates/controllerserver-security-context-constraint.yaml @@ -6,14 +6,14 @@ metadata: allowPrivilegedContainer: true allowHostDirVolumePlugin: true -{{- if .Values.hostNetwork }} +{{- if or .Values.hostNetwork .Values.pluginConfig.mountProtocol.allowNfsFailback .Values.pluginConfig.mountProtocol.useNfs }} allowHostNetwork: true {{- end }} allowedVolumeTypes: - hostPath - secret readOnlyRootFilesystem: false - +allowHostPorts: true runAsUser: type: RunAsAny seLinuxContext: diff --git a/charts/csi-wekafsplugin/templates/controllerserver-serviceaccount.yaml b/charts/csi-wekafsplugin/templates/controllerserver-serviceaccount.yaml index 26a9794bf..aad9fd923 100644 --- a/charts/csi-wekafsplugin/templates/controllerserver-serviceaccount.yaml +++ b/charts/csi-wekafsplugin/templates/controllerserver-serviceaccount.yaml @@ -1,7 +1,9 @@ apiVersion: v1 kind: ServiceAccount +{{- if .Values.imagePullSecret}} imagePullSecrets: - - name: {{ .Release.Name }}-creds + - name: {{ .Values.imagePullSecret }} +{{- end }} metadata: name: {{ .Release.Name }}-controller namespace: {{ .Release.Namespace }} diff --git a/charts/csi-wekafsplugin/templates/controllerserver-statefulset.yaml b/charts/csi-wekafsplugin/templates/controllerserver-statefulset.yaml index b15ee8ab5..8ffd18cfe 100755 --- a/charts/csi-wekafsplugin/templates/controllerserver-statefulset.yaml +++ b/charts/csi-wekafsplugin/templates/controllerserver-statefulset.yaml @@ -23,15 +23,15 @@ spec: annotations: prometheus.io/scrape: 'true' prometheus.io/path: '/metrics' - prometheus.io/port: '{{ .Values.metrics.port | default 9090 }}' + prometheus.io/port: '{{ .Values.metrics.controllerPort | default 9090 }},{{ .Values.metrics.provisionerPort | default 9091 }},{{ .Values.metrics.resizerPort | default 9092 }},{{ .Values.metrics.snapshotterPort | default 9093 }}' {{- end }} spec: {{- if .Values.nodeSelector }} nodeSelector: {{ toYaml .Values.nodeSelector | nindent 8}} {{- end }} serviceAccountName: {{ .Release.Name }}-controller - {{- if .Values.hostNetwork }} - hostNetwork: {{ .Values.hostNetwork }} + {{- if or .Values.hostNetwork .Values.pluginConfig.mountProtocol.useNfs .Values.pluginConfig.mountProtocol.allowNfsFailback}} + hostNetwork: true {{- end }} containers: - name: csi-attacher @@ -192,7 +192,7 @@ spec: {{- end }} {{- if .Values.metrics.enabled }} - "--enablemetrics" - - "--metricsport={{ .Values.metrics.port | default 9090 }}" + - "--metricsport={{ .Values.metrics.controllerPort | default 9090 }}" {{- end }} {{- if .Values.pluginConfig.allowInsecureHttps }} - "--allowinsecurehttps" @@ -215,12 +215,27 @@ spec: - "--concurrency.createSnapshot={{ .Values.controller.concurrency.createSnapshot | default "1" }}" - "--concurrency.deleteSnapshot={{ .Values.controller.concurrency.deleteSnapshot | default "1" }}" {{- end }} + {{- if .Values.pluginConfig.mountProtocol.useNfs | default false }} + - "--usenfs" + {{- end }} + {{- if .Values.pluginConfig.mountProtocol.allowNfsFailback | default false }} + - "--allownfsfailback" + {{- end }} + {{- if .Values.pluginConfig.mountProtocol.interfaceGroupName }} + - "--interfacegroupname={{ .Values.pluginConfig.mountProtocol.interfaceGroupName }}" + {{- end }} + {{- if .Values.pluginConfig.mountProtocol.clientGroupName }} + - "--clientgroupname={{ .Values.pluginConfig.mountProtocol.clientGroupName }}" + {{- end }} + {{- if .Values.pluginConfig.mountProtocol.nfsProtocolVersion }} + - "--nfsprotocolversion={{ .Values.pluginConfig.mountProtocol.nfsProtocolVersion | toString}}" + {{- end }} ports: - containerPort: 9898 name: healthz protocol: TCP {{- if .Values.metrics.enabled }} - - containerPort: {{ .Values.metrics.port }} + - containerPort: {{ .Values.metrics.controllerPort | default 9090 }} name: metrics protocol: TCP {{- end }} @@ -249,6 +264,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: KUBE_NODE_IP_ADDRESS + valueFrom: + fieldRef: + fieldPath: status.hostIP volumeMounts: - mountPath: /csi name: socket-dir diff --git a/charts/csi-wekafsplugin/templates/nodeserver-daemonset.yaml b/charts/csi-wekafsplugin/templates/nodeserver-daemonset.yaml index e438de56b..d3edd40d8 100644 --- a/charts/csi-wekafsplugin/templates/nodeserver-daemonset.yaml +++ b/charts/csi-wekafsplugin/templates/nodeserver-daemonset.yaml @@ -17,7 +17,7 @@ spec: annotations: prometheus.io/scrape: 'true' prometheus.io/path: '/metrics' - prometheus.io/port: '{{ .Values.metrics.port | default 9090 }}' + prometheus.io/port: '{{ .Values.metrics.nodePort | default 9090 }}' {{- end }} spec: {{- if (eq .Values.selinuxSupport "mixed")}} @@ -39,15 +39,15 @@ spec: {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{- end }} - {{- if .Values.hostNetwork }} - hostNetwork: {{ .Values.hostNetwork }} + {{- if or .Values.hostNetwork .Values.pluginConfig.mountProtocol.useNfs .Values.pluginConfig.mountProtocol.allowNfsFailback}} + hostNetwork: true {{- end }} initContainers: - name: init volumeMounts: - mountPath: /etc/nodeinfo name: nodeinfo - image: {{ .Values.images.nodeinfo }} + image: "{{ .Values.images.nodeinfo }}:v{{ .Values.images.csidriverTag}}" imagePullPolicy: IfNotPresent securityContext: # This doesn't need to run as root. @@ -58,8 +58,9 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - args: + command: - bash + args: - -c - kubectl get node $NODENAME -o json | jq '.metadata' > /etc/nodeinfo/metadata containers: @@ -86,7 +87,7 @@ spec: {{- end }} {{- if .Values.metrics.enabled }} - "--enablemetrics" - - "--metricsport={{ .Values.metrics.port | default 9090 }}" + - "--metricsport={{ .Values.metrics.nodePort | default 9090 }}" {{- end }} {{- if .Values.pluginConfig.allowInsecureHttps }} - "--allowinsecurehttps" @@ -106,12 +107,27 @@ spec: - "--concurrency.nodePublishVolume={{ .Values.node.concurrency.nodePublishVolume | default "1" }}" - "--concurrency.nodeUnpublishVolume={{ .Values.node.concurrency.nodeUnpublishVolume | default "1" }}" {{- end }} + {{- if .Values.pluginConfig.mountProtocol.useNfs | default false }} + - "--usenfs" + {{- end }} + {{- if .Values.pluginConfig.mountProtocol.allowNfsFailback | default false }} + - "--allownfsfailback" + {{- end }} + {{- if .Values.pluginConfig.mountProtocol.interfaceGroupName }} + - "--interfacegroupname={{ .Values.pluginConfig.mountProtocol.interfaceGroupName }}" + {{- end }} + {{- if .Values.pluginConfig.mountProtocol.clientGroupName }} + - "--clientgroupname={{ .Values.pluginConfig.mountProtocol.clientGroupName }}" + {{- end }} + {{- if .Values.pluginConfig.mountProtocol.nfsProtocolVersion }} + - "--nfsprotocolversion={{ .Values.pluginConfig.mountProtocol.nfsProtocolVersion | toString}}" + {{- end }} ports: - containerPort: 9899 name: healthz protocol: TCP {{- if .Values.metrics.enabled }} - - containerPort: {{ .Values.metrics.port }} + - containerPort: {{ .Values.metrics.nodePort }} name: metrics protocol: TCP {{- end }} @@ -136,6 +152,10 @@ spec: value: {{ required "Provide CSI Driver Dynamic Volume Creation Path" .Values.dynamicProvisionPath }} - name: X_CSI_MODE value: node + - name: KUBE_NODE_IP_ADDRESS + valueFrom: + fieldRef: + fieldPath: status.hostIP volumeMounts: - mountPath: /csi name: socket-dir diff --git a/charts/csi-wekafsplugin/templates/nodeserver-security-context-constraint.yaml b/charts/csi-wekafsplugin/templates/nodeserver-security-context-constraint.yaml index 1334e7c9d..153a3c4db 100644 --- a/charts/csi-wekafsplugin/templates/nodeserver-security-context-constraint.yaml +++ b/charts/csi-wekafsplugin/templates/nodeserver-security-context-constraint.yaml @@ -6,7 +6,7 @@ metadata: allowPrivilegedContainer: true allowHostDirVolumePlugin: true -{{- if .Values.hostNetwork }} +{{- if or .Values.hostNetwork .Values.pluginConfig.mountProtocol.allowNfsFailback .Values.pluginConfig.mountProtocol.useNfs }} allowHostNetwork: true {{- end }} allowedVolumeTypes: diff --git a/charts/csi-wekafsplugin/templates/nodeserver-serviceaccount.yaml b/charts/csi-wekafsplugin/templates/nodeserver-serviceaccount.yaml index 0e89c8443..c6b620b4d 100644 --- a/charts/csi-wekafsplugin/templates/nodeserver-serviceaccount.yaml +++ b/charts/csi-wekafsplugin/templates/nodeserver-serviceaccount.yaml @@ -1,7 +1,9 @@ apiVersion: v1 kind: ServiceAccount +{{- if .Values.imagePullSecret}} imagePullSecrets: - - name: {{ .Release.Name }}-creds + - name: {{ .Values.imagePullSecret }} +{{- end }} metadata: name: {{ .Release.Name }}-node namespace: {{ .Release.Namespace }} diff --git a/charts/csi-wekafsplugin/values.schema.json b/charts/csi-wekafsplugin/values.schema.json index 28343d061..ab077d062 100644 --- a/charts/csi-wekafsplugin/values.schema.json +++ b/charts/csi-wekafsplugin/values.schema.json @@ -91,6 +91,9 @@ "hostNetwork": { "type": "boolean" }, + "imagePullSecret": { + "type": "string" + }, "images": { "type": "object", "properties": { @@ -141,10 +144,13 @@ "metrics": { "type": "object", "properties": { + "controllerPort": { + "type": "integer" + }, "enabled": { "type": "boolean" }, - "port": { + "nodePort": { "type": "integer" }, "provisionerPort": { @@ -226,6 +232,26 @@ "fsGroupPolicy": { "type": "string" }, + "mountProtocol": { + "type": "object", + "properties": { + "allowNfsFailback": { + "type": "boolean" + }, + "clientGroupName": { + "type": "string" + }, + "interfaceGroupName": { + "type": "string" + }, + "nfsProtocolVersion": { + "type": "string" + }, + "useNfs": { + "type": "boolean" + } + } + }, "mutuallyExclusiveMountOptions": { "type": "array", "items": { diff --git a/charts/csi-wekafsplugin/values.yaml b/charts/csi-wekafsplugin/values.yaml index 2bd6e5c7e..c14adf2b0 100644 --- a/charts/csi-wekafsplugin/values.yaml +++ b/charts/csi-wekafsplugin/values.yaml @@ -5,26 +5,29 @@ dynamicProvisionPath: "csi-volumes" # -- Name of the driver (and provisioner) csiDriverName: "csi.weka.io" # -- CSI driver version -csiDriverVersion: &csiDriverVersion 2.4.1 +csiDriverVersion: &csiDriverVersion 2.4.2-SNAPSHOT.99.90161ea images: # -- CSI liveness probe sidecar image URL - livenessprobesidecar: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 + livenessprobesidecar: registry.k8s.io/sig-storage/livenessprobe:v2.14.0 # -- CSI attacher sidecar image URL - attachersidecar: registry.k8s.io/sig-storage/csi-attacher:v4.5.0 + attachersidecar: registry.k8s.io/sig-storage/csi-attacher:v4.7.0 # -- CSI provisioner sidecar image URL - provisionersidecar: registry.k8s.io/sig-storage/csi-provisioner:v4.0.0 + provisionersidecar: registry.k8s.io/sig-storage/csi-provisioner:v5.1.0 # -- CSI registrar sidercar - registrarsidecar: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0 + registrarsidecar: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.12.0 # -- CSI resizer sidecar image URL - resizersidecar: registry.k8s.io/sig-storage/csi-resizer:v1.9.3 + resizersidecar: registry.k8s.io/sig-storage/csi-resizer:v1.12.0 # -- CSI snapshotter sidecar image URL - snapshottersidecar: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.3 + snapshottersidecar: registry.k8s.io/sig-storage/csi-snapshotter:v8.1.0 # -- CSI nodeinfo sidecar image URL, used for reading node metadata - nodeinfo: quay.io/weka.io/kubectl-sidecar:v1.29.2-1 + nodeinfo: quay.io/weka.io/csi-wekafs # -- CSI driver main image URL csidriver: quay.io/weka.io/csi-wekafs # -- CSI driver tag csidriverTag: *csiDriverVersion +# -- image pull secret required for image download. Must have permissions to access all images above. +# Should be used in case of private registry that requires authentication +imagePullSecret: "" # -- Tolerations for all CSI driver components globalPluginTolerations: &globalPluginTolerations - key: node-role.kubernetes.io/master @@ -96,18 +99,20 @@ kubeletPath: "/var/lib/kubelet" metrics: # -- Enable Prometheus Metrics enabled: true - # -- Metrics port - port: 9090 + # -- Metrics port for Controller Server + controllerPort: 9090 # -- Provisioner metrics port provisionerPort: 9091 # -- Resizer metrics port resizerPort: 9092 # -- Snapshotter metrics port snapshotterPort: 9093 + # -- Metrics port for Node Serer + nodePort: 9094 # -- Tracing URL (For Jaeger tracing engine / OpenTelemetry), optional # @ignore tracingUrl: "" -# -- Set to true to use host networking +# -- Set to true to use host networking. Will be always set to true when using NFS mount protocol hostNetwork: false pluginConfig: # -- CSI Driver support for fsGroupPolicy, may be either "File" or "None". Default is "File" @@ -140,3 +145,15 @@ pluginConfig: snapshotVolumesWithoutQuotaEnforcement: false mutuallyExclusiveMountOptions: - "readcache,writecache,coherent,forcedirect" + - "sync,async" + mountProtocol: + # -- Use NFS transport for mounting Weka filesystems, off by default + useNfs: false + # -- Allow Failback to NFS transport if Weka client fails to mount filesystem using native protocol + allowNfsFailback: false + # -- Specify name of NFS interface group to use for mounting Weka filesystems. If not set, first NFS interface group will be used + interfaceGroupName: "" + # -- Specify existing client group name for NFS configuration. If not set, "WekaCSIPluginClients" group will be created + clientGroupName: "" + # -- Specify NFS protocol version to use for mounting Weka filesystems. Default is "4.1", consult Weka documentation for supported versions + nfsProtocolVersion: "4.1" diff --git a/cmd/wekafsplugin/main.go b/cmd/wekafsplugin/main.go index 54c8a9271..f4ab75b8b 100644 --- a/cmd/wekafsplugin/main.go +++ b/cmd/wekafsplugin/main.go @@ -91,6 +91,11 @@ var ( maxConcurrentNodeUnpublishVolumeReqs = flag.Int64("concurrency.nodeUnpublishVolume", 1, "Maximum concurrent NodeUnpublishVolume requests") grpcRequestTimeoutSeconds = flag.Int("grpcrequesttimeoutseconds", 30, "Time out requests waiting in queue after X seconds") allowProtocolContainers = flag.Bool("allowprotocolcontainers", false, "Allow protocol containers to be used for mounting filesystems") + allowNfsFailback = flag.Bool("allownfsfailback", false, "Allow NFS failback") + useNfs = flag.Bool("usenfs", false, "Use NFS for mounting volumes") + interfaceGroupName = flag.String("interfacegroupname", "", "Name of the NFS interface group to use for mounting volumes") + clientGroupName = flag.String("clientgroupname", "", "Name of the NFS client group to use for managing NFS permissions") + nfsProtocolVersion = flag.String("nfsprotocolversion", "4.1", "NFS protocol version to use for mounting volumes") // Set by the build process version = "" ) @@ -217,6 +222,11 @@ func handle() { *maxConcurrentNodeUnpublishVolumeReqs, *grpcRequestTimeoutSeconds, *allowProtocolContainers, + *allowNfsFailback, + *useNfs, + *interfaceGroupName, + *clientGroupName, + *nfsProtocolVersion, ) driver, err := wekafs.NewWekaFsDriver( *driverName, *nodeID, *endpoint, *maxVolumesPerNode, version, *debugPath, csiMode, *selinuxSupport, config) diff --git a/docs/NFS.md b/docs/NFS.md new file mode 100644 index 000000000..40fddfbf2 --- /dev/null +++ b/docs/NFS.md @@ -0,0 +1,151 @@ +# Weka CSI Plugin with NFS transport + +## Overview +Although using native WekaFS driver as the underlying storage connectivity layer is recommended way to use WekaFS with Kubernetes, +it is also possible to use the Weka CSI Plugin over NFS transport. +This allows you to use WekaFS as a storage backend for your Kubernetes cluster without the need to install the Weka client on each Kubernetes node. + +### Benefits of using Weka CSI Plugin with NFS transport +- **Simplified deployment**: No need to install the Weka client on each Kubernetes node +- **Interoperability**: Use Weka CSI Plugin on nodes where the Weka client is not yet installed, or is not currently supported +- **Flexibility**: Use Weka CSI Plugin with NFS transport for specific use-cases, while using the native WekaFS driver for other use-cases +- **Performance**: Pods are mounted across multiple IPs on the same NFS interface group, maximizing performance and simplifying management +- **Ease of migration**: Use Weka CSI Plugin with NFS transport as a stepping stone to migrate to the native WekaFS driver. + After deployment of the Weka client on all nodes, you can switch to the native WekaFS driver without changing the storage configuration + by simply rebooting the node. + +### Limitations and Constraints +- **Performance**: NFS transport is not as performant as the native WekaFS driver and it is not recommended for high-performance workloads +- **Feature Parity**: Some features and capabilities of the native WekaFS driver are not available when using the Weka CSI Plugin with NFS transport +- **Complexity**: NFS transport requires additional configuration on the Weka cluster, and may require additional networking configuration on the Kubernetes cluster +- **Interoperability**: Same Kubernetes node cannot use both NFS and WekaFS transport at the same time +- **Migration**: Migrating from NFS transport to WekaFS transport requires rebooting the Kubernetes nodes (after Weka client deployment) +- **Network Configuration**: NFS interface group IP addresses must be accessible from the Kubernetes cluster nodes +- **Security**: NFS transport is less secure than the native WekaFS driver, and may require additional security considerations +- **QoS**: QoS is not supported for NFS transport + +### Host Network Mode +Weka CSI Plugin will automatically install in `hostNetwork` mode when using NFS transport. +Since hostNetwork mode is required for NFS transport, the `hostNetwork` parameter in the `values.yaml` file is ignored in such case. + +### Security Considerations +- The Weka CSI Plugin with NFS transport uses NFSv4.1 protocol to connect to the Weka cluster. +- Support for Kerberos authentication is not available in this version of Weka CSI Plugin. +- It is recommended to use NFS transport only in secure and trusted networks. + +## Interoperability with WekaFS driver +The Weka CSI Plugin with NFS transport is fully interoperable with the native WekaFS driver. + +This means that you can use both WekaFS transport and NFS in the same Kubernetes cluster, +and even for publishing the same volume to different pods using different transport layers (from different nodes). +However, only one transport layer can be used on a single node at a time. + +### Mount options +Mount options for the NFS transport are set automatically by the Weka CSI Plugin. When custom mount options are used in storage class, +the Weka CSI Plugin will translate them to NFS alternatives. Unknown or unsupported mount options will be ignored. + +### QoS and Performance +QoS is not supported for NFS transport. Performance is limited by the NFS protocol and the network configuration. + +### Switching from NFS to WekaFS transport +To switch between NFS and WekaFS transport, you need to: +1. Install the Weka client on Kubernetes node +2. Reboot the Kubernetes node + +After the node is rebooted, the Weka CSI Plugin will automatically switch to using the WekaFS transport. +Existing volumes can be reattached to the pods without any changes. + +## Prerequisites +Those are the minimum prerequisites for using Weka CSI Plugin with NFS transport: + +- Weka cluster must be installed and configured +- NFS protocol must be configured on the Weka cluster +- NFS interface group must be created on the Weka cluster +- NFS interface group IP addresses must be accessible from the Kubernetes cluster nodes + +> **WARNING:** When multiple NFS interface groups are defined on Weka clusters, +> the `pluginConfig.mountProtocol.interfaceGroupName` parameter must be set to the desired NFS interface group name in the `values.yaml` file. +> If the parameter is not set, an arbitrary NFS interface group will be used, that could potentially cause performance or networking issues. + +> **NOTE**: NFS Client group called `WekaCSIPluginClients` is created automatically by the Weka CSI Plugin. +> Then, upon each volume creation or publishing, the Kubernetes node IP address is added to the NFS Client group automatically. +> +> Although, adding the node IP addresses one by one is the most secure way to configure the NFS Client group, this could become cumbersome in large deployments. +> In such case, using a network range (CIDR) is recommended. +> You may predefine the NFS Client group with a network range (CIDR) in the Weka cluster, and then use the `pluginConfig.mountProtocol.nfsClientGroupName` +> parameter in the `values.yaml` file to specify the NFS Client group name. + +## Way of Operation +The Weka CSI Plugin with NFS transport operates in the following way: +Upon start of the Weka CSI Plugin, the plugin will: +1. Check if the Weka client is installed on the Kubernetes node +2. If client is not set up, the plugin will check whether NFS failback is enabled +3. If NFS failback is enabled, the plugin will use NFS transport for volume provisioning and publishing +4. If NFS failback is disabled, the plugin will not start and will log an error message. + Refer to the [Installation](#installation) section for enabling NFS failback. + +Once NFS mode is enabled, the Weka CSI Plugin will use NFS transport for all volume operations. +In such case, upon any volume create or publish request, the Weka CSI Plugin will: +1. Connect to Weka cluster API and fetch interface groups (and their IP addresses) + If interface group name is specified in the `values.yaml` file, + the plugin will use the specified interface group, otherwise an arbitraty interface group will be used. +2. Ensure that Client Group is created on the Weka cluster. + If the Client Group is not created, the plugin will create it. + > **NOTE:** If client group name is specified in the `values.yaml` file, the plugin will use the specified client group name, + > otherwise `WekaCSIPluginClients` client group will be used. +3. Determine the node IP address facing the inteface group IP addresses. This will be done by checking the network configuration of the node + Then, the Weka CSI plugin will issue a UDP connection towards one of the IP addresses of the interface group, + The source IP address of the connection will be determined by the plugin and will be used as the `node IP address`. +4. Ensure that the `node IP address` is added to the Client Group. + If the node IP address is not added, the plugin will add it to the Client Group. + If client group already has the node IP address (or it has a matching CIDR definition), the plugin will skip this step. + > **EXAMPLE:** If the node IP address is `192.168.100.1` and the client group is defined with a network range `192.168.100.0/255.255.255.0`, + > node IP address will not be added +5. Identify the filesystem name to be mounted, either from StorageClass parameters (provisioning), + or from Volume Handle (for publishing an existing volume). +6. Ensure that NFS permission exists for the Client Group to access the filesystem. + If the permission is not set, the plugin will set it. If the permission is already set, the plugin will skip this step. +7. Pick up a random IP address from the selected NFS interface group. + This IP address will be used for mounting the filesystem. +8. Perform NFS mount operation on the Kubernetes node using the selected IP address and the filesystem name. +9. Rest of the operations will be performed in a similar way as with the native WekaFS driver. + +## NFS Permissions Required for Weka CSI Plugin +The Weka CSI Plugin requires AND will set the following NFS permissions on the Weka cluster: +1. **Client Group**: `WekaCSIPluginClients` (or custom client group name if set in the `values.yaml` file) +2. **Filesystem**: The filesystem name to be mounted +3. **Path**: `/` (root of the filesystem) +4. **Type**: `RW` +5. **Priority**: No priority set +6. **Supported Versions**: `V4` +7. **User Squash**: `None` +8. **Authentication Types**: `NONE`, `SYS` + +> **WARNING:** Weka NFS servers will evaluate permissions based on the order of the permissions list. +> If multiple permissions matching the IP address of the Kubernetes node and the filesystem are set, a conflict might occur. +> Hence, it is **highly recommended** not creating additional permissions for the same filesystem +> Also, if multiple client groups are used, it is highly recommended to make sure that IP addresses are not overlapping between client groups. + +## Installation +By default, Weka CSI Plugin components will not start unless Weka driver is not detected on Kubernetes node. +This is to prevent a potential misconfiguration where volumes are attempted to be provisioned or published on node while no Weka client is installed. + +To enable NFS transport, Weka CSI plugin must be explicitly configured for using NFS failback. +This is done by setting the `pluginConfig.mountProtocol.allowNfsFailback` parameter to `true` in the `values.yaml` file. + +The parameter `pluginConfig.mountProtocol.useNfs` enforces the use of NFS transport even if Weka client is installed on the node, +and recommended to be set to `true` ONLY for testing. + +Follow the [Helm installation instructions](./charts/csi-wekafsplugin/README.md) to install the Weka CSI Plugin. +Most of the installation steps are the same as for the native WekaFS driver, however, additional parameters should be set in the `values.yaml` file, +or passed as command line arguments to the `helm install` command. + +This is the example Helm install command for using NFS transport: +```console +helm upgrade csi-wekafs -n csi-wekafs --create-namespace --install csi-wekafs/csi-wekafsplugin csi-wekafs\ +--set logLevel=6 \ +--set pluginConfig.mountProtocol.alloeNfsFailback=true \ +--set pluginConfig.allowInsecureHttps=true \ +[ --set pluginConfig.mountProtocol.interfaceGroupName=MyIntefaceGroup \ ] # optional, recommended if multiple interface groups are defined +[ --set pluginConfig.mountProtocol.clientGroupName=MyClientGroup \ ] # optional, recommended if client group is predefined +``` diff --git a/examples/common/csi-wekafs-api-secret.yaml b/examples/common/csi-wekafs-api-secret.yaml index c1c3b6b30..6a8468351 100644 --- a/examples/common/csi-wekafs-api-secret.yaml +++ b/examples/common/csi-wekafs-api-secret.yaml @@ -15,7 +15,7 @@ data: # It is recommended to configure at least 2 management endpoints (cluster backend nodes), or a load-balancer if used # e.g. 172.31.15.113:14000,172.31.12.91:14000 endpoints: MTcyLjMxLjQxLjU0OjE0MDAwLDE3Mi4zMS40Ny4xNTI6MTQwMDAsMTcyLjMxLjM4LjI1MDoxNDAwMCwxNzIuMzEuNDcuMTU1OjE0MDAwLDE3Mi4zMS4zMy45MToxNDAwMCwxNzIuMzEuMzguMTU1OjE0MDAwCg== - # protocol to use for API connection (may be either http or https, base64-encoded) + # protocol to use for API connection (may be either http or https, base64-encoded. NOTE: since Weka 4.3.0, HTTPS is mandatory) scheme: aHR0cA== # for multiple clusters setup, set specific container name rather than attempt to identify it automatically localContainerName: "" @@ -24,3 +24,7 @@ data: # maybe either (true/false), base64-encoded # NOTE: if a load balancer is used to access the cluster API, leave this setting as "false" autoUpdateEndpoints: ZmFsc2U= + # When using HTTPS connection and self-signed or untrusted certificates, provide a CA certificate in PEM format, base64-encoded + # caCertificate: + caCertificate: "" + diff --git a/examples/dynamic_directory/csi-statefulset-on-dir-api.yaml b/examples/dynamic_directory/csi-statefulset-on-dir-api.yaml new file mode 100644 index 000000000..b54823fbd --- /dev/null +++ b/examples/dynamic_directory/csi-statefulset-on-dir-api.yaml @@ -0,0 +1,44 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-wekafs-test-statefulset-on-dir-api + labels: + app: "csi-wekafs-test-statefulset-on-dir-api" +spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Delete + whenScaled: Retain + replicas: 100 + selector: + matchLabels: + kubernetes.io/os: linux + template: + metadata: + labels: + kubernetes.io/os: linux + app: "csi-wekafs-test-statefulset-on-dir-api" + spec: + terminationGracePeriodSeconds: 1 + # make sure that pod is scheduled only on node having weka CSI node running + nodeSelector: + topology.csi.weka.io/global: "true" + containers: + - name: my-frontend + image: busybox + volumeMounts: + - mountPath: "/data" + name: csi-wekafs-dir-api + command: ["/bin/sh"] + args: ["-c", "while true; do echo `date` hello >> /data/`hostname`.txt; sleep 10;done"] + volumeClaimTemplates: + - metadata: + name: csi-wekafs-dir-api + labels: + app: "csi-wekafs-test-statefulset-on-dir-api" + spec: + accessModes: [ "ReadWriteMany" ] + storageClassName: storageclass-wekafs-dir-api + resources: + requests: + storage: 1Gi + serviceName: "csi-wekafs-test-statefulset-on-dir-api" diff --git a/go.mod b/go.mod index 6b6766c91..18f44402b 100644 --- a/go.mod +++ b/go.mod @@ -1,50 +1,60 @@ module github.com/wekafs/csi-wekafs -go 1.22.0 - -toolchain go1.22.4 +go 1.22.5 require ( github.com/container-storage-interface/spec v1.10.0 github.com/google/go-querystring v1.1.0 github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.7.0 - github.com/kubernetes-csi/csi-lib-utils v0.18.1 + github.com/kubernetes-csi/csi-lib-utils v0.19.0 + github.com/pkg/errors v0.9.1 github.com/pkg/xattr v0.4.10 - github.com/prometheus/client_golang v1.19.1 + github.com/prometheus/client_golang v1.20.2 github.com/rs/zerolog v1.33.0 github.com/showa-93/go-mask v0.6.2 - go.opentelemetry.io/otel v1.28.0 + github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/otel v1.29.0 go.opentelemetry.io/otel/exporters/jaeger v1.17.0 - go.opentelemetry.io/otel/sdk v1.28.0 - go.opentelemetry.io/otel/trace v1.28.0 - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 - golang.org/x/sync v0.7.0 - google.golang.org/grpc v1.65.0 + go.opentelemetry.io/otel/sdk v1.29.0 + go.opentelemetry.io/otel/trace v1.29.0 + golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 + golang.org/x/sync v0.8.0 + google.golang.org/grpc v1.66.0 google.golang.org/protobuf v1.34.2 - k8s.io/apimachinery v0.30.3 + k8s.io/apimachinery v0.31.0 k8s.io/helm v2.17.0+incompatible - k8s.io/mount-utils v0.30.3 + k8s.io/mount-utils v0.31.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/kr/text v0.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/runc v1.1.13 // indirect + github.com/opencontainers/runtime-spec v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/common v0.57.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - golang.org/x/net v0.27.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + go.opentelemetry.io/otel/metric v1.29.0 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.17.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + k8s.io/utils v0.0.0-20240821151609-f90d01438635 // indirect ) diff --git a/go.sum b/go.sum index e7a564d21..c36b7427c 100644 --- a/go.sum +++ b/go.sum @@ -4,15 +4,21 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/container-storage-interface/spec v1.10.0 h1:YkzWPV39x+ZMTa6Ax2czJLLwpryrQ+dPesB34mrRMXA= github.com/container-storage-interface/spec v1.10.0/go.mod h1:DtUvaQszPml1YJfIK7c00mlv6/g4wNMLanLgiUbKFRI= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -24,8 +30,16 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/kubernetes-csi/csi-lib-utils v0.18.1 h1:vpg1kbQ6lFVCz7mY71zcqVE7W0GAQXXBoFfHvbW3gdw= -github.com/kubernetes-csi/csi-lib-utils v0.18.1/go.mod h1:PIcn27zmbY0KBue4JDdZVfDF56tjcS3jKroZPi+pMoY= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-csi/csi-lib-utils v0.19.0 h1:3sT8mL9+St2acyrEtuR7CQ5L78GR4lgsb+sfon9tGfA= +github.com/kubernetes-csi/csi-lib-utils v0.19.0/go.mod h1:lBuMKvoyd8c3EG+itmnVWApLDHnLkU7ibxxZSPuOw0M= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -36,68 +50,85 @@ github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9Kou github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/opencontainers/runc v1.1.13 h1:98S2srgG9vw0zWcDpFMn5TRrh8kLxa/5OFUstuUhmRs= +github.com/opencontainers/runc v1.1.13/go.mod h1:R016aXacfp/gwQBYw2FDGa9m+n6atbLWrYY8hNMT/sA= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA= github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= +github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY= +github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/showa-93/go-mask v0.6.2 h1:sJEUQRpbxUoMTfBKey5K9hCg+eSx5KIAZFT7pa1LXbM= github.com/showa-93/go-mask v0.6.2/go.mod h1:aswIj007gm0EPAzOGES9ACy1jDm3QT08/LPSClMp410= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4= go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf h1:liao9UHurZLtiEwBgT9LMOnKYsHze6eA6w1KQCMVN2Q= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed h1:J6izYgfBXAI3xTKLgxzTmUltdYaLsuBxFCgDHWJ/eXg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= -k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= +k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= k8s.io/helm v2.17.0+incompatible h1:Bpn6o1wKLYqKM3+Osh8e+1/K2g/GsQJ4F4yNF2+deao= k8s.io/helm v2.17.0+incompatible/go.mod h1:LZzlS4LQBHfciFOurYBFkCMTaZ0D1l+p0teMg7TSULI= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/mount-utils v0.30.3 h1:8Z3wSW5+GSvGNtlDhtoZrBCKLMIf5z/9tf8pie+G06s= -k8s.io/mount-utils v0.30.3/go.mod h1:9sCVmwGLcV1MPvbZ+rToMDnl1QcGozy+jBPd0MsQLIo= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/mount-utils v0.31.0 h1:o+a+n6gyZ7MGc6bIERU3LeFTHbLDBiVReaDpWlJotUE= +k8s.io/mount-utils v0.31.0/go.mod h1:HV/VYBUGqYUj4vt82YltzpWvgv8FPg0G9ItyInT3NPU= +k8s.io/utils v0.0.0-20240821151609-f90d01438635 h1:2wThSvJoW/Ncn9TmQEYXRnevZXi2duqHWf5OX9S3zjI= +k8s.io/utils v0.0.0-20240821151609-f90d01438635/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= diff --git a/pkg/wekafs/apiclient/apiclient.go b/pkg/wekafs/apiclient/apiclient.go index 253200f8e..5aff66973 100644 --- a/pkg/wekafs/apiclient/apiclient.go +++ b/pkg/wekafs/apiclient/apiclient.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -63,6 +64,7 @@ type ApiClient struct { CompatibilityMap *WekaCompatibilityMap clientHash uint32 hostname string + NfsInterfaceGroups map[string]*InterfaceGroup } type ApiEndPoint struct { @@ -89,9 +91,22 @@ func (e *ApiEndPoint) String() string { } func NewApiClient(ctx context.Context, credentials Credentials, allowInsecureHttps bool, hostname string) (*ApiClient, error) { + logger := log.Ctx(ctx) tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: allowInsecureHttps}, } + useCustomCACert := credentials.CaCertificate != "" + if useCustomCACert { + var caCertPool *x509.CertPool + if pool, err := x509.SystemCertPool(); err != nil { + caCertPool = x509.NewCertPool() + } else { + caCertPool = pool + } + caCertPool.AppendCertsFromPEM([]byte(credentials.CaCertificate)) + tr.TLSClientConfig.RootCAs = caCertPool + } + a := &ApiClient{ Mutex: sync.Mutex{}, client: &http.Client{ @@ -105,10 +120,11 @@ func NewApiClient(ctx context.Context, credentials Credentials, allowInsecureHtt CompatibilityMap: &WekaCompatibilityMap{}, hostname: hostname, actualApiEndpoints: make(map[string]*ApiEndPoint), + NfsInterfaceGroups: make(map[string]*InterfaceGroup), } a.resetDefaultEndpoints(ctx) - log.Ctx(ctx).Trace().Bool("insecure_skip_verify", allowInsecureHttps).Msg("Creating new API client") + logger.Trace().Bool("insecure_skip_verify", allowInsecureHttps).Bool("custom_ca_cert", useCustomCACert).Msg("Creating new API client") a.clientHash = a.generateHash() return a, nil } @@ -731,21 +747,21 @@ type ApiResponse struct { // ApiObject generic interface of API object of any type (FileSystem, Quota, etc.) type ApiObject interface { - GetType() string - GetBasePath(a *ApiClient) string - GetApiUrl(a *ApiClient) string - EQ(other ApiObject) bool - getImmutableFields() []string - String() string + GetType() string // returns the type of the object + GetBasePath(a *ApiClient) string // returns the base path of objects of this type (plural) + GetApiUrl(a *ApiClient) string // returns the full URL of the object consisting of base path and object UID + EQ(other ApiObject) bool // a way to compare objects and check if they are the same + getImmutableFields() []string // provides a list of fields that are used for comparison in EQ() + String() string // returns a string representation of the object } // ApiObjectRequest interface that describes a request for an ApiObject CRUD operation type ApiObjectRequest interface { - getRequiredFields() []string - hasRequiredFields() bool - getRelatedObject() ApiObject - getApiUrl(a *ApiClient) string - String() string + getRequiredFields() []string // returns a list of fields that are mandatory for the object for creation + hasRequiredFields() bool // checks if all mandatory fields are filled in + getRelatedObject() ApiObject // returns the type of object that is being requested + getApiUrl(a *ApiClient) string // returns the full URL of the object consisting of base path and object UID + String() string // returns a string representation of the object request } type Credentials struct { @@ -756,6 +772,7 @@ type Credentials struct { Endpoints []string LocalContainerName string AutoUpdateEndpoints bool + CaCertificate string } func (c *Credentials) String() string { diff --git a/pkg/wekafs/apiclient/interfacegroup.go b/pkg/wekafs/apiclient/interfacegroup.go new file mode 100644 index 000000000..09b75e7dd --- /dev/null +++ b/pkg/wekafs/apiclient/interfacegroup.go @@ -0,0 +1,210 @@ +package apiclient + +import ( + "context" + "errors" + "fmt" + "github.com/google/uuid" + "github.com/rs/zerolog/log" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/helm/pkg/urlutil" + "os" + "sort" +) + +type InterfaceGroupType string + +const ( + InterfaceGroupTypeNFS InterfaceGroupType = "NFS" + InterfaceGroupTypeSMB InterfaceGroupType = "SMB" +) + +type InterfaceGroup struct { + SubnetMask string `json:"subnet_mask"` + Name string `json:"name"` + Uid uuid.UUID `json:"uid"` + Ips []string `json:"ips"` + AllowManageGids bool `json:"allow_manage_gids"` + Type InterfaceGroupType `json:"type"` + Gateway string `json:"gateway"` + Status string `json:"status"` +} + +func (i *InterfaceGroup) String() string { + return fmt.Sprintln("InterfaceGroup ", i.Name, "Uid:", i.Uid.String(), "type:", i.Type, "status:", i.Status) +} + +func (i *InterfaceGroup) getImmutableFields() []string { + return []string{"Name", "Gateway", "SubnetMask", "Type"} +} + +func (i *InterfaceGroup) GetType() string { + return "interfaceGroup" +} + +func (i *InterfaceGroup) GetBasePath(client *ApiClient) string { + return "interfaceGroups" +} + +func (i *InterfaceGroup) GetApiUrl(client *ApiClient) string { + url, err := urlutil.URLJoin(i.GetBasePath(client), i.Uid.String()) + if err == nil { + return url + } + return "" +} + +func (i *InterfaceGroup) EQ(other ApiObject) bool { + return ObjectsAreEqual(i, other) +} + +func (i *InterfaceGroup) getInterfaceGroupType() InterfaceGroupType { + return i.Type +} + +func (i *InterfaceGroup) isNfs() bool { + return i.getInterfaceGroupType() == InterfaceGroupTypeNFS +} + +func (i *InterfaceGroup) isSmb() bool { + return i.getInterfaceGroupType() == InterfaceGroupTypeSMB +} + +// GetIpAddress returns a single IP address based on hostname, so for same server, always same IP address will be returned +// This is useful for NFS mount, where we need to have same IP address for same server +// TODO: this could be further optimized in future to avoid a situation where multiple servers are not evenly distributed +// and some IPs are getting more load than others. Could be done, for example, by random selection of IP address etc. +func (i *InterfaceGroup) GetIpAddress(ctx context.Context) (string, error) { + logger := log.Ctx(ctx) + if i == nil { + return "", errors.New("interface group is nil") + } + if len(i.Ips) == 0 { + return "", errors.New("no IP addresses found for interface group") + } + hostname, err := os.Hostname() + if err != nil { + return "", err + } + if hostname == "" { + hostname = "localhost" + } + idx := hashString(hostname, len(i.Ips)) + logger.Debug().Int("index", idx).Str("hostname", hostname).Int("ips", len(i.Ips)).Msg("Selected IP address based on hostname") + return i.Ips[idx], nil +} + +func (i *InterfaceGroup) GetRandomIpAddress(ctx context.Context) (string, error) { + logger := log.Ctx(ctx) + if i == nil { + return "", errors.New("interface group is nil") + } + if len(i.Ips) == 0 { + return "", errors.New("no IP addresses found for interface group") + } + idx := rand.Intn(len(i.Ips)) + ip := i.Ips[idx] + logger.Debug().Str("ip", ip).Msg("Selected random IP address") + return ip, nil +} + +func (a *ApiClient) GetInterfaceGroups(ctx context.Context, interfaceGroups *[]InterfaceGroup) error { + ig := &InterfaceGroup{} + + err := a.Get(ctx, ig.GetBasePath(a), nil, interfaceGroups) + if err != nil { + return err + } + return nil +} + +func (a *ApiClient) GetInterfaceGroupsByType(ctx context.Context, groupType InterfaceGroupType, interfaceGroups *[]InterfaceGroup) error { + res := &[]InterfaceGroup{} + err := a.GetInterfaceGroups(ctx, res) + if err != nil { + return nil + } + for _, ig := range *res { + if ig.getInterfaceGroupType() == groupType { + *interfaceGroups = append(*interfaceGroups, ig) + } + } + return nil +} + +func (a *ApiClient) GetInterfaceGroupByUid(ctx context.Context, uid uuid.UUID, interfaceGroup *InterfaceGroup) error { + ig := &InterfaceGroup{ + Uid: uid, + } + err := a.Get(ctx, ig.GetApiUrl(a), nil, interfaceGroup) + if err != nil { + return err + } + return nil +} + +func (a *ApiClient) fetchNfsInterfaceGroup(ctx context.Context, name *string, useDefault bool) error { + igs := &[]InterfaceGroup{} + err := a.GetInterfaceGroupsByType(ctx, InterfaceGroupTypeNFS, igs) + if err != nil { + return errors.Join(errors.New("failed to fetch nfs interface groups"), err) + } + if len(*igs) == 0 { + return errors.New("no nfs interface groups found") + } + if name != nil { + for _, ig := range *igs { + if ig.Name == *name { + a.NfsInterfaceGroups[*name] = &ig + } + } + } else if useDefault { + a.NfsInterfaceGroups["default"] = &(*igs)[0] + } + + ig := &InterfaceGroup{} + if name != nil { + ig = a.NfsInterfaceGroups[*name] + } else { + ig = a.NfsInterfaceGroups["default"] + } + if ig == nil { + return errors.New("no nfs interface group found") + } + + if len(ig.Ips) == 0 { + return errors.New("no IP addresses found for nfs interface group") + } + // Make sure the IPs are always sorted + sort.Strings(ig.Ips) + return nil +} + +func (a *ApiClient) GetNfsInterfaceGroup(ctx context.Context, name *string) *InterfaceGroup { + igName := "default" + if name != nil { + igName = *name + } + _, ok := a.NfsInterfaceGroups[igName] + if !ok { + err := a.fetchNfsInterfaceGroup(ctx, name, true) + if err != nil { + return nil + } + } + return a.NfsInterfaceGroups[igName] +} + +// GetNfsMountIp returns the IP address of the NFS interface group to be used for NFS mount +// TODO: need to do it much more sophisticated way to distribute load +func (a *ApiClient) GetNfsMountIp(ctx context.Context, interfaceGroupName *string) (string, error) { + ig := a.GetNfsInterfaceGroup(ctx, interfaceGroupName) + if ig == nil { + return "", errors.New("no NFS interface group found") + } + if ig.Ips == nil || len(ig.Ips) == 0 { + return "", errors.New("no IP addresses found for NFS interface group") + } + + return ig.GetRandomIpAddress(ctx) +} diff --git a/pkg/wekafs/apiclient/nfs.go b/pkg/wekafs/apiclient/nfs.go new file mode 100644 index 000000000..adb74ec37 --- /dev/null +++ b/pkg/wekafs/apiclient/nfs.go @@ -0,0 +1,749 @@ +package apiclient + +import ( + "context" + "encoding/json" + "errors" + "fmt" + qs "github.com/google/go-querystring/query" + "github.com/google/uuid" + "github.com/rs/zerolog/log" + "go.opentelemetry.io/otel" + "golang.org/x/exp/slices" + "k8s.io/helm/pkg/urlutil" + "strconv" + "strings" +) + +type NfsPermissionType string +type NfsPermissionSquashMode string +type NfsClientGroupRuleType string +type NfsVersionString string + +func (n NfsVersionString) String() string { + return string(n) +} +func (n NfsVersionString) AsOption() string { + return strings.TrimLeft(n.String(), "V") +} + +func (n NfsVersionString) AsWeka() NfsVersionString { + return NfsVersionString(strings.Split(n.String(), ".")[0]) +} + +type NfsAuthType string + +const ( + NfsPermissionTypeReadWrite NfsPermissionType = "RW" + NfsPermissionTypeReadOnly NfsPermissionType = "RO" + NfsPermissionSquashModeNone NfsPermissionSquashMode = "none" + NfsPermissionSquashModeRoot NfsPermissionSquashMode = "root" + NfsPermissionSquashModeAll NfsPermissionSquashMode = "all" + NfsClientGroupRuleTypeDNS NfsClientGroupRuleType = "DNS" + NfsClientGroupRuleTypeIP NfsClientGroupRuleType = "IP" + NfsVersionV3 NfsVersionString = "V3" + NfsVersionV4 NfsVersionString = "V4" + NfsAuthTypeNone NfsAuthType = "NONE" + NfsAuthTypeSys NfsAuthType = "SYS" + NfsAuthTypeKerberos5 NfsAuthType = "KRB5" + NfsClientGroupName = "WekaCSIPluginClients" +) + +type NfsPermission struct { + GroupId string `json:"group_id,omitempty" url:"-"` + PrivilegedPort bool `json:"privileged_port,omitempty" url:"-"` + SupportedVersions []NfsVersionString `json:"supported_versions,omitempty" url:"-"` + Id string `json:"id,omitempty" url:"-"` + ObsDirect bool `json:"obs_direct,omitempty" url:"-"` + AnonUid string `json:"anon_uid,omitempty" url:"-"` + ManageGids bool `json:"manage_gids,omitempty" url:"-"` + CustomOptions string `json:"custom_options,omitempty" url:"-"` + Filesystem string `json:"filesystem" url:"-"` + Uid uuid.UUID `json:"uid,omitempty" url:"-"` + Group string `json:"group" url:"-"` + NfsClientGroupId string `json:"NfsClientGroup_id,omitempty" url:"-"` + PermissionType NfsPermissionType `json:"permission_type,omitempty" url:"-"` + MountOptions string `json:"mount_options,omitempty" url:"-"` + Path string `json:"path,omitempty" url:"-"` + SquashMode NfsPermissionSquashMode `json:"squash_mode,omitempty" url:"-"` + RootSquashing bool `json:"root_squashing,omitempty" url:"-"` + AnonGid string `json:"anon_gid,omitempty" url:"-"` + EnableAuthTypes []NfsAuthType `json:"enable_auth_types,omitempty" url:"-"` +} + +func (n *NfsPermission) GetType() string { + return "nfsPermission" +} + +func (n *NfsPermission) GetBasePath(a *ApiClient) string { + return "nfs/permissions" +} + +func (n *NfsPermission) GetApiUrl(a *ApiClient) string { + url, err := urlutil.URLJoin(n.GetBasePath(a), n.Uid.String()) + if err != nil { + return url + } + return "" +} + +func (n *NfsPermission) EQ(other ApiObject) bool { + return ObjectsAreEqual(n, other) +} + +func (n *NfsPermission) getImmutableFields() []string { + return []string{"Group", "Filesystem", "SupportedVersions", "PermissionType", "Path", "SquashMode"} +} + +func (n *NfsPermission) String() string { + return fmt.Sprintln("NfsPermission Uid:", n.Uid.String(), "NfsClientGroup:", n.Group, "path:", n.Path) +} + +func (n *NfsPermission) IsEligibleForCsi() bool { + return n.RootSquashing == false && slices.Contains(n.SupportedVersions, "V4") && + n.PermissionType == NfsPermissionTypeReadWrite && + n.SquashMode == NfsPermissionSquashModeNone +} + +func (a *ApiClient) GetNfsPermissions(ctx context.Context, fsUid uuid.UUID, permissions *[]NfsPermission) error { + n := &NfsPermission{} + + err := a.Get(ctx, n.GetBasePath(a), nil, permissions) + if err != nil { + return err + } + return nil +} + +func (a *ApiClient) FindNfsPermissionsByFilter(ctx context.Context, query *NfsPermission, resultSet *[]NfsPermission) error { + op := "FindNfsPermissionsByFilter" + ctx, span := otel.Tracer(TracerName).Start(ctx, op) + defer span.End() + ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) + ret := &[]NfsPermission{} + q, _ := qs.Values(query) + err := a.Get(ctx, query.GetBasePath(a), q, ret) + if err != nil { + return err + } + for _, r := range *ret { + if r.EQ(query) { + *resultSet = append(*resultSet, r) + } + } + return nil +} + +// GetNfsPermissionByFilter expected to return exactly one result of FindNfsPermissionsByFilter (error) +func (a *ApiClient) GetNfsPermissionByFilter(ctx context.Context, query *NfsPermission) (*NfsPermission, error) { + rs := &[]NfsPermission{} + err := a.FindNfsPermissionsByFilter(ctx, query, rs) + if err != nil { + return &NfsPermission{}, err + } + if *rs == nil || len(*rs) == 0 { + return &NfsPermission{}, ObjectNotFoundError + } + if len(*rs) > 1 { + return &NfsPermission{}, MultipleObjectsFoundError + } + result := &(*rs)[0] + return result, nil +} + +func (a *ApiClient) GetNfsPermissionsByFilesystemName(ctx context.Context, fsName string, permissions *[]NfsPermission) error { + query := &NfsPermission{Path: fsName} + return a.FindNfsPermissionsByFilter(ctx, query, permissions) +} + +func (a *ApiClient) GetNfsPermissionByUid(ctx context.Context, uid uuid.UUID) (*NfsPermission, error) { + query := &NfsPermission{Uid: uid} + return a.GetNfsPermissionByFilter(ctx, query) +} + +type NfsPermissionCreateRequest struct { + Filesystem string `json:"filesystem"` + Group string `json:"group"` + Path string `json:"path"` + PermissionType NfsPermissionType `json:"permission_type"` + SquashMode NfsPermissionSquashMode `json:"squash_mode"` + AnonUid int `json:"anon_uid"` + AnonGid int `json:"anon_gid"` + ObsDirect *bool `json:"obs_direct,omitempty"` + SupportedVersions *[]string `json:"supported_versions,omitempty"` + Priority int `json:"priority"` + EnableAuthTypes []NfsAuthType `json:"enable_auth_types"` +} + +func (qc *NfsPermissionCreateRequest) getApiUrl(a *ApiClient) string { + return qc.getRelatedObject().GetApiUrl(a) +} +func (qc *NfsPermissionCreateRequest) getRelatedObject() ApiObject { + return &NfsPermission{ + GroupId: qc.Group, + } +} + +func (qc *NfsPermissionCreateRequest) getRequiredFields() []string { + return []string{"Filesystem", "Group", "Path", "PermissionType", "SquashMode", "SupportedVersions"} +} +func (qc *NfsPermissionCreateRequest) hasRequiredFields() bool { + return ObjectRequestHasRequiredFields(qc) +} + +func (qc *NfsPermissionCreateRequest) String() string { + return fmt.Sprintln("NfsPermissionCreateRequest(FS:", qc.Filesystem) +} + +func (a *ApiClient) CreateNfsPermission(ctx context.Context, r *NfsPermissionCreateRequest, p *NfsPermission) error { + op := "CreateNfsPermission" + ctx, span := otel.Tracer(TracerName).Start(ctx, op) + defer span.End() + ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) + if !r.hasRequiredFields() { + return RequestMissingParams + } + payload, err := json.Marshal(r) + if err != nil { + return err + } + + err = a.Post(ctx, r.getRelatedObject().GetBasePath(a), &payload, nil, p) + return err +} + +func EnsureNfsPermission(ctx context.Context, fsName string, group string, version NfsVersionString, apiClient *ApiClient) error { + perm := &NfsPermission{ + SupportedVersions: []NfsVersionString{version.AsWeka()}, + AnonUid: strconv.Itoa(65534), + AnonGid: strconv.Itoa(65534), + Filesystem: fsName, + Group: group, + PermissionType: NfsPermissionTypeReadWrite, + Path: "/", + SquashMode: NfsPermissionSquashModeNone, + } + _, err := apiClient.GetNfsPermissionByFilter(ctx, perm) + if err != nil { + if err == ObjectNotFoundError { + req := &NfsPermissionCreateRequest{ + Filesystem: fsName, + Group: group, + Path: "/", + PermissionType: NfsPermissionTypeReadWrite, + SquashMode: NfsPermissionSquashModeNone, + AnonGid: 65534, + AnonUid: 65534, + SupportedVersions: &[]string{string(NfsVersionV4)}, + } + return apiClient.CreateNfsPermission(ctx, req, perm) + } + } + return err +} + +type NfsClientGroup struct { + Uid uuid.UUID `json:"uid,omitempty" url:"-"` + Rules []NfsClientGroupRule `json:"rules,omitempty" url:"-"` + Id string `json:"id,omitempty" url:"-"` + Name string `json:"name,omitempty" url:"name,omitempty"` +} + +func (g *NfsClientGroup) GetType() string { + return "clientGroup" +} + +func (g *NfsClientGroup) GetBasePath(a *ApiClient) string { + return "nfs/clientGroups" +} + +func (g *NfsClientGroup) GetApiUrl(a *ApiClient) string { + url, err := urlutil.URLJoin(g.GetBasePath(a), g.Uid.String()) + if err == nil { + return url + } + return "" +} + +func (g *NfsClientGroup) EQ(other ApiObject) bool { + return ObjectsAreEqual(g, other) +} + +func (g *NfsClientGroup) getImmutableFields() []string { + return []string{"Name"} +} + +func (g *NfsClientGroup) String() string { + return fmt.Sprintln("NfsClientGroup name:", g.Name) +} + +func (a *ApiClient) GetNfsClientGroups(ctx context.Context, clientGroups *[]NfsClientGroup) error { + cg := &NfsClientGroup{} + + err := a.Get(ctx, cg.GetBasePath(a), nil, clientGroups) + if err != nil { + return err + } + return nil +} + +func (a *ApiClient) FindNfsClientGroupsByFilter(ctx context.Context, query *NfsClientGroup, resultSet *[]NfsClientGroup) error { + op := "FindNfsClientGroupsByFilter" + ctx, span := otel.Tracer(TracerName).Start(ctx, op) + defer span.End() + ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) + logger := log.Ctx(ctx) + logger.Trace().Str("client_group_query", query.String()).Msg("Finding client groups by filter") + ret := &[]NfsClientGroup{} + q, _ := qs.Values(query) + err := a.Get(ctx, query.GetBasePath(a), q, ret) + if err != nil { + return err + } + for _, r := range *ret { + if r.EQ(query) { + *resultSet = append(*resultSet, r) + } + } + return nil +} + +// GetNfsClientGroupByFilter expected to return exactly one result of FindNfsClientGroupsByFilter (error) +func (a *ApiClient) GetNfsClientGroupByFilter(ctx context.Context, query *NfsClientGroup) (*NfsClientGroup, error) { + op := "GetNfsClientGroupByFilter" + ctx, span := otel.Tracer(TracerName).Start(ctx, op) + defer span.End() + ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) + logger := log.Ctx(ctx) + rs := &[]NfsClientGroup{} + err := a.FindNfsClientGroupsByFilter(ctx, query, rs) + logger.Trace().Str("client_group", query.String()).Msg("Getting client group by filter") + if err != nil { + return &NfsClientGroup{}, err + } + if *rs == nil || len(*rs) == 0 { + return &NfsClientGroup{}, ObjectNotFoundError + } + if len(*rs) > 1 { + return &NfsClientGroup{}, MultipleObjectsFoundError + } + result := &(*rs)[0] + return result, nil +} + +func (a *ApiClient) GetNfsClientGroupByName(ctx context.Context, name string) (*NfsClientGroup, error) { + query := &NfsClientGroup{Name: name} + return a.GetNfsClientGroupByFilter(ctx, query) +} + +func (a *ApiClient) GetNfsClientGroupByUid(ctx context.Context, uid uuid.UUID, cg *NfsClientGroup) error { + ret := &NfsClientGroup{ + Uid: uid, + } + err := a.Get(ctx, ret.GetApiUrl(a), nil, cg) + if err != nil { + switch t := err.(type) { + case *ApiNotFoundError: + return ObjectNotFoundError + case *ApiBadRequestError: + for _, c := range t.ApiResponse.ErrorCodes { + if c == "ClientGroupDoesNotExistException" { + return ObjectNotFoundError + } + } + default: + return err + } + } + return nil + +} + +func (a *ApiClient) CreateNfsClientGroup(ctx context.Context, r *NfsClientGroupCreateRequest, fs *NfsClientGroup) error { + op := "CreateNfsClientGroup" + ctx, span := otel.Tracer(TracerName).Start(ctx, op) + defer span.End() + ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) + if !r.hasRequiredFields() { + return RequestMissingParams + } + payload, err := json.Marshal(r) + if err != nil { + return err + } + + err = a.Post(ctx, r.getRelatedObject().GetBasePath(a), &payload, nil, fs) + return err +} + +func (a *ApiClient) EnsureCsiPluginNfsClientGroup(ctx context.Context, clientGroupName string) (*NfsClientGroup, error) { + op := "EnsureCsiPluginNfsClientGroup" + ctx, span := otel.Tracer(TracerName).Start(ctx, op) + defer span.End() + ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) + logger := log.Ctx(ctx) + var ret *NfsClientGroup + if clientGroupName == "" { + clientGroupName = NfsClientGroupName + } + logger.Trace().Str("client_group_name", clientGroupName).Msg("Getting client group by name") + ret, err := a.GetNfsClientGroupByName(ctx, clientGroupName) + if err != nil { + if err != ObjectNotFoundError { + logger.Error().Err(err).Msg("Failed to get client group by name") + return ret, err + } else { + logger.Trace().Str("client_group_name", clientGroupName).Msg("Existing client group not found, creating client group") + err = a.CreateNfsClientGroup(ctx, NewNfsClientGroupCreateRequest(clientGroupName), ret) + } + } + return ret, nil +} + +type NfsClientGroupCreateRequest struct { + Name string `json:"name"` +} + +func (fsc *NfsClientGroupCreateRequest) getApiUrl(a *ApiClient) string { + return fsc.getRelatedObject().GetBasePath(a) +} + +func (fsc *NfsClientGroupCreateRequest) getRequiredFields() []string { + return []string{"Name"} +} + +func (fsc *NfsClientGroupCreateRequest) hasRequiredFields() bool { + return ObjectRequestHasRequiredFields(fsc) +} +func (fsc *NfsClientGroupCreateRequest) getRelatedObject() ApiObject { + return &NfsClientGroup{} +} + +func (fsc *NfsClientGroupCreateRequest) String() string { + return fmt.Sprintln("NfsClientGroupCreateRequest(name:", fsc.Name) +} + +func NewNfsClientGroupCreateRequest(name string) *NfsClientGroupCreateRequest { + return &NfsClientGroupCreateRequest{ + Name: name, + } +} + +type NfsClientGroupDeleteRequest struct { + Uid uuid.UUID `json:"-"` +} + +func (cgd *NfsClientGroupDeleteRequest) getApiUrl(a *ApiClient) string { + return cgd.getRelatedObject().GetApiUrl(a) +} + +func (cgd *NfsClientGroupDeleteRequest) getRelatedObject() ApiObject { + return &NfsClientGroup{Uid: cgd.Uid} +} + +func (cgd *NfsClientGroupDeleteRequest) getRequiredFields() []string { + return []string{"Uid"} +} + +func (cgd *NfsClientGroupDeleteRequest) hasRequiredFields() bool { + return ObjectRequestHasRequiredFields(cgd) +} + +func (cgd *NfsClientGroupDeleteRequest) String() string { + return fmt.Sprintln("NfsClientGroupDeleteRequest(uid:", cgd.Uid) +} + +func (a *ApiClient) DeleteNfsClientGroup(ctx context.Context, r *NfsClientGroupDeleteRequest) error { + op := "DeleteNfsClientGroup" + ctx, span := otel.Tracer(TracerName).Start(ctx, op) + defer span.End() + ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) + if !r.hasRequiredFields() { + return RequestMissingParams + } + apiResponse := &ApiResponse{} + err := a.Delete(ctx, r.getApiUrl(a), nil, nil, apiResponse) + if err != nil { + switch t := err.(type) { + case *ApiNotFoundError: + return ObjectNotFoundError + case *ApiBadRequestError: + for _, c := range t.ApiResponse.ErrorCodes { + if c == "FilesystemDoesNotExistException" { + return ObjectNotFoundError + } + } + } + } + return nil +} + +type NfsClientGroupRule struct { + NfsClientGroupUid uuid.UUID `json:"-" url:"-"` + Type NfsClientGroupRuleType `json:"type,omitempty" url:"-"` + Uid uuid.UUID `json:"uid,omitempty" url:"-"` + Rule string `json:"rule,omitempty" url:"-"` + Id string `json:"id,omitempty" url:"-"` +} + +func (r *NfsClientGroupRule) GetType() string { + return "rules" +} + +func (r *NfsClientGroupRule) GetBasePath(a *ApiClient) string { + ncgUrl := (&NfsClientGroup{Uid: r.Uid}).GetApiUrl(a) + url, err := urlutil.URLJoin(ncgUrl, r.GetType()) + if err != nil { + return "" + } + return url +} + +func (r *NfsClientGroupRule) GetApiUrl(a *ApiClient) string { + url, err := urlutil.URLJoin(r.GetBasePath(a), r.Uid.String()) + if err != nil { + return url + } + return "" +} + +func (r *NfsClientGroupRule) EQ(other ApiObject) bool { + return ObjectsAreEqual(r, other) +} + +func (r *NfsClientGroupRule) IsSupersetOf(other *NfsClientGroupRule) bool { + if r.IsIPRule() && other.IsIPRule() { + n1 := r.GetNetwork() + n2 := other.GetNetwork() + return n1.ContainsIPAddress(n2.IP.String()) + } + return false +} + +func (r *NfsClientGroupRule) getImmutableFields() []string { + return []string{"Rule"} +} + +func (r *NfsClientGroupRule) String() string { + return fmt.Sprintln("NfsClientGroupRule Uid:", r.Uid.String(), "clientGroupUid:", r.NfsClientGroupUid.String(), + "type:", r.Type, "rule", r.Rule) +} + +func (r *NfsClientGroupRule) IsIPRule() bool { + return r.Type == NfsClientGroupRuleTypeIP +} + +func (r *NfsClientGroupRule) IsDNSRule() bool { + return r.Type == NfsClientGroupRuleTypeDNS +} + +func (r *NfsClientGroupRule) GetNetwork() *Network { + if !r.IsIPRule() { + return nil + } + n, err := parseNetworkString(r.Rule) + if err != nil { + return nil + } + return n +} + +func (r *NfsClientGroupRule) IsEligibleForIP(ip string) bool { + network := r.GetNetwork() + if network == nil { + return false + } + return network.ContainsIPAddress(ip) +} + +func (a *ApiClient) GetNfsClientGroupRules(ctx context.Context, clientGroupName string, rules *[]NfsClientGroupRule) error { + op := "GetNfsClientGroupRules" + ctx, span := otel.Tracer(TracerName).Start(ctx, op) + defer span.End() + ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) + cg, err := a.EnsureCsiPluginNfsClientGroup(ctx, clientGroupName) + if err != nil { + return err + } + copiedRules := make([]NfsClientGroupRule, len(cg.Rules)) + copy(copiedRules, cg.Rules) + *rules = copiedRules + return nil +} + +func (a *ApiClient) FindNfsClientGroupRulesByFilter(ctx context.Context, query *NfsClientGroupRule, resultSet *[]NfsClientGroupRule) error { + op := "FindNfsClientGroupRulesByFilter" + ctx, span := otel.Tracer(TracerName).Start(ctx, op) + defer span.End() + ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) + logger := log.Ctx(ctx) + + // this is different that in other functions since we don't have /rules entry point for GET + // so we need to get the client group first + logger.Trace().Str("client_group_uid", query.NfsClientGroupUid.String()).Msg("Getting client group") + cg := &NfsClientGroup{} + err := a.GetNfsClientGroupByUid(ctx, query.NfsClientGroupUid, cg) + if cg == nil || err != nil { + return err + } + ret := cg.Rules + + for _, r := range ret { + if r.EQ(query) { + *resultSet = append(*resultSet, r) + } else if r.IsSupersetOf(query) { + // if we have a rule that covers the IP address by bigger network segment, also add it + *resultSet = append(*resultSet, r) + } + } + return nil +} + +func (a *ApiClient) GetNfsClientGroupRuleByFilter(ctx context.Context, rule *NfsClientGroupRule) (*NfsClientGroupRule, error) { + op := "GetNfsClientGroupRuleByFilter" + ctx, span := otel.Tracer(TracerName).Start(ctx, op) + defer span.End() + ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) + rs := &[]NfsClientGroupRule{} + err := a.FindNfsClientGroupRulesByFilter(ctx, rule, rs) + if err != nil { + return &NfsClientGroupRule{}, err + } + if *rs == nil || len(*rs) == 0 { + return &NfsClientGroupRule{}, ObjectNotFoundError + } + if len(*rs) > 1 { + return &NfsClientGroupRule{}, MultipleObjectsFoundError + } + result := &(*rs)[0] + return result, nil +} + +type NfsClientGroupRuleCreateRequest struct { + NfsClientGroupUid uuid.UUID `json:"-"` + Type NfsClientGroupRuleType `json:"-"` + Hostname string `json:"dns,omitempty"` + Ip string `json:"ip,omitempty"` +} + +func (r *NfsClientGroupRuleCreateRequest) getType() string { + return "rules" +} + +func (r *NfsClientGroupRuleCreateRequest) getApiUrl(a *ApiClient) string { + ret, err := urlutil.URLJoin(r.getRelatedObject().GetApiUrl(a), r.getType()) + if err != nil { + return "" + } + return ret +} + +func (r *NfsClientGroupRuleCreateRequest) getRequiredFields() []string { + return []string{"Type"} +} + +func (r *NfsClientGroupRuleCreateRequest) hasRequiredFields() bool { + return ObjectRequestHasRequiredFields(r) +} + +func (r *NfsClientGroupRuleCreateRequest) getRelatedObject() ApiObject { + return &NfsClientGroup{Uid: r.NfsClientGroupUid} +} + +func (r *NfsClientGroupRuleCreateRequest) String() string { + return fmt.Sprintln("NfsClientGroupRuleCreateRequest(NfsClientGroupUid:", r.NfsClientGroupUid, "Type:", r.Type) +} + +func (r *NfsClientGroupRuleCreateRequest) AsRule() string { + if r.Type == NfsClientGroupRuleTypeDNS { + return r.Hostname + } + return r.Ip +} + +func NewNfsClientGroupRuleCreateRequest(cgUid uuid.UUID, ruleType NfsClientGroupRuleType, rule string) *NfsClientGroupRuleCreateRequest { + + ret := &NfsClientGroupRuleCreateRequest{ + NfsClientGroupUid: cgUid, + Type: ruleType, + } + if ruleType == NfsClientGroupRuleTypeDNS { + ret.Hostname = rule + } else if ruleType == NfsClientGroupRuleTypeIP { + net, err := parseNetworkString(rule) + if err != nil { + return nil + } + ret.Ip = net.AsNfsRule() + } + return ret +} + +func (a *ApiClient) CreateNfsClientGroupRule(ctx context.Context, r *NfsClientGroupRuleCreateRequest, rule *NfsClientGroupRule) error { + op := "CreateNfsClientGroupRule" + ctx, span := otel.Tracer(TracerName).Start(ctx, op) + defer span.End() + ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) + logger := log.Ctx(ctx) + logger.Trace().Str("client_group_rule", r.String()).Msg("Creating client group rule") + + if !r.hasRequiredFields() { + return RequestMissingParams + } + + payload, err := json.Marshal(r) + if err != nil { + return err + } + + err = a.Post(ctx, r.getApiUrl(a), &payload, nil, rule) + return err +} + +func (a *ApiClient) EnsureNfsClientGroupRuleForIp(ctx context.Context, cg *NfsClientGroup, ip string) error { + if cg == nil { + return errors.New("NfsClientGroup is nil") + } + r, err := parseNetworkString(ip) + if err != nil { + return err + } + + q := &NfsClientGroupRule{Type: NfsClientGroupRuleTypeIP, Rule: r.AsNfsRule(), NfsClientGroupUid: cg.Uid} + + rule, err := a.GetNfsClientGroupRuleByFilter(ctx, q) + if err != nil { + if err == ObjectNotFoundError { + req := NewNfsClientGroupRuleCreateRequest(cg.Uid, q.Type, q.Rule) + return a.CreateNfsClientGroupRule(ctx, req, rule) + } + } + return err +} + +func (a *ApiClient) EnsureNfsPermissions(ctx context.Context, ip string, fsName string, version NfsVersionString, clientGroupName string) error { + op := "EnsureNfsPermissions" + ctx, span := otel.Tracer(TracerName).Start(ctx, op) + defer span.End() + ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) + logger := log.Ctx(ctx) + logger.Debug().Str("ip", ip).Str("filesystem", fsName).Str("client_group_name", clientGroupName).Msg("Ensuring NFS permissions") + // Ensure client group + logger.Trace().Msg("Ensuring CSI Plugin NFS Client Group") + cg, err := a.EnsureCsiPluginNfsClientGroup(ctx, clientGroupName) + if err != nil { + logger.Error().Err(err).Msg("Failed to ensure NFS client group") + return err + } + + // Ensure client group rule + logger.Trace().Str("ip_address", ip).Msg("Ensuring NFS Client Group Rule for IP") + err = a.EnsureNfsClientGroupRuleForIp(ctx, cg, ip) + if err != nil { + return err + } + // Ensure NFS permission + logger.Trace().Str("filesystem", fsName).Str("client_group", cg.Name).Msg("Ensuring NFS Export for client group") + err = EnsureNfsPermission(ctx, fsName, cg.Name, version, a) + return err +} diff --git a/pkg/wekafs/apiclient/nfs_test.go b/pkg/wekafs/apiclient/nfs_test.go new file mode 100644 index 000000000..7067ac2d2 --- /dev/null +++ b/pkg/wekafs/apiclient/nfs_test.go @@ -0,0 +1,344 @@ +package apiclient + +import ( + "context" + "flag" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/util/rand" + "testing" +) + +var creds Credentials +var endpoint string +var fsName string + +var client *ApiClient + +func TestMain(m *testing.M) { + flag.StringVar(&endpoint, "api-endpoint", "vm49-1723969301909816-0.lan:14000", "API endpoint for tests") + flag.StringVar(&creds.Username, "api-username", "admin", "API username for tests") + flag.StringVar(&creds.Password, "api-password", "AAbb1234", "API password for tests") + flag.StringVar(&creds.Organization, "api-org", "Root", "API org for tests") + flag.StringVar(&creds.HttpScheme, "api-scheme", "https", "API scheme for tests") + flag.StringVar(&fsName, "fs-name", "default", "Filesystem name for tests") + flag.Parse() + m.Run() +} + +func GetApiClientForTest(t *testing.T) *ApiClient { + creds.Endpoints = []string{endpoint} + if client == nil { + apiClient, err := NewApiClient(context.Background(), creds, true, "test") + if err != nil { + t.Fatalf("Failed to create API client: %v", err) + } + if apiClient == nil { + t.Fatalf("Failed to create API client") + } + if err := apiClient.Login(context.Background()); err != nil { + t.Fatalf("Failed to login: %v", err) + } + client = apiClient + } + return client +} + +// +//func TestGetNfsPermissions(t *testing.T) { +// apiClient := GetApiClientForTest(t) +// +// var permissions []NfsPermission +// +// req := &NfsPermissionCreateRequest{ +// Filesystem: fsName, +// Group: "group1", +// } +// p := &NfsPermission{} +// err := apiClient.CreateNfsPermission(context.Background(), &NfsPermissionCreateRequest{}, p) +// assert.NoError(t, err) +// assert.NotZero(t, p.Uid) +// +// err := apiClient.GetNfsPermissions(context.Background(), &permissions) +// assert.NoError(t, err) +// assert.NotEmpty(t, permissions) +//} +// +//func TestFindNfsPermissionsByFilter(t *testing.T) { +// apiClient := GetApiClientForTest(t) +// query := &NfsPermission{Filesystem: "fs1"} +// var resultSet []NfsPermission +// err := apiClient.FindNfsPermissionsByFilter(context.Background(), query, &resultSet) +// assert.NoError(t, err) +// assert.NotEmpty(t, resultSet) +//} +// +//func TestGetNfsPermissionByFilter(t *testing.T) { +// apiClient := GetApiClientForTest(t) +// +// query := &NfsPermission{Filesystem: "fs1"} +// result, err := apiClient.GetNfsPermissionByFilter(context.Background(), query) +// assert.NoError(t, err) +// assert.NotNil(t, result) +//} +// +//func TestGetNfsPermissionsByFilesystemName(t *testing.T) { +// apiClient := GetApiClientForTest(t) +// +// +// var permissions []NfsPermission +// err := apiClient.GetNfsPermissionsByFilesystemName(context.Background(), "fs1", &permissions) +// assert.NoError(t, err) +// assert.NotEmpty(t, permissions) +//} +// +//func TestGetNfsPermissionByUid(t *testing.T) { +// apiClient := GetApiClientForTest(t) +// server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// w.WriteHeader(http.StatusOK) +// w.Write([]byte(`{"filesystem": "fs1", "group": "group1"}`)) +// })) +// defer server.Close() +// +// +// uid := uuid.New() +// result, err := apiClient.GetNfsPermissionByUid(context.Background(), uid) +// assert.NoError(t, err) +// assert.NotNil(t, result) +//} +// +//func TestCreateNfsPermission(t *testing.T) { +// apiClient := GetApiClientForTest(t) +// +// req := &NfsPermissionCreateRequest{ +// Filesystem: "fs1", +// Group: "group1", +// SquashMode: NfsPermissionSquashModeNone, +// AnonUid: 1000, +// AnonGid: 1000, +// EnableAuthTypes: []NfsAuthType{NfsAuthTypeSys}, +// } +// var perm NfsPermission +// err := apiClient.CreateNfsPermission(context.Background(), req, &perm) +// assert.NoError(t, err) +// assert.NotNil(t, perm) +//} +// +//func TestEnsureNfsPermission(t *testing.T) { +// apiClient := GetApiClientForTest(t) +// server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// w.WriteHeader(http.StatusOK) +// w.Write([]byte(`{"filesystem": "fs1", "group": "group1"}`)) +// })) +// defer server.Close() +// +// +// err := EnsureNfsPermission(context.Background(), "fs1", "group1", apiClient) +// assert.NoError(t, err) +//} + +func TestNfsClientGroup(t *testing.T) { + apiClient := GetApiClientForTest(t) + + var clientGroups []NfsClientGroup + var cg = &NfsClientGroup{ + Uid: uuid.New(), + } + // Test GetApiUrl + assert.NotEmpty(t, cg.GetApiUrl(apiClient)) + assert.Contains(t, cg.GetApiUrl(apiClient), cg.Uid.String()) + + // Test EQ + cg1 := &NfsClientGroup{ + Name: "test", + } + + cg2 := &NfsClientGroup{ + Name: "test", + } + assert.True(t, cg1.EQ(cg2)) + + // Test GetBasePath + assert.NotEmpty(t, cg.GetBasePath(apiClient)) + + // Test Create + cgName := rand.String(10) + err := apiClient.CreateNfsClientGroup(context.Background(), &NfsClientGroupCreateRequest{Name: cgName}, cg) + assert.NotEmpty(t, cg.Uid) + assert.NoError(t, err) + assert.Equal(t, cgName, cg.Name) + assert.Empty(t, cg.Rules) + + // Test GetGroups + err = apiClient.GetNfsClientGroups(context.Background(), &clientGroups) + assert.NoError(t, err) + assert.NotEmpty(t, clientGroups) + + // Test GetGroupByUid + uid := cg.Uid + err = apiClient.GetNfsClientGroupByUid(context.Background(), uid, cg) + assert.NoError(t, err) + assert.Equal(t, cgName, cg.Name) + assert.NotEmpty(t, cg.Uid) + + // Test GetGroupByName + name := cg.Name + cg, err = apiClient.GetNfsClientGroupByName(context.Background(), name) + assert.NoError(t, err) + assert.Equal(t, cgName, cg.Name) + assert.NotEmpty(t, cg.Uid) + + // Test Delete + r := &NfsClientGroupDeleteRequest{Uid: cg.Uid} + err = apiClient.DeleteNfsClientGroup(context.Background(), r) + assert.NoError(t, err) + err = apiClient.GetNfsClientGroups(context.Background(), &clientGroups) + assert.NoError(t, err) + for _, r := range clientGroups { + if r.Uid == cg.Uid { + t.Errorf("Failed to delete group") + } + } +} + +func TestEnsureCsiPluginNfsClientGroup(t *testing.T) { + apiClient := GetApiClientForTest(t) + result, err := apiClient.EnsureCsiPluginNfsClientGroup(context.Background(), NfsClientGroupName) + assert.NoError(t, err) + assert.NotNil(t, result) +} + +func TestNfsClientGroupRules(t *testing.T) { + apiClient := GetApiClientForTest(t) + cg, err := apiClient.EnsureCsiPluginNfsClientGroup(context.Background(), NfsClientGroupName) + assert.NoError(t, err) + assert.NotNil(t, cg) + + // Test Create + r := &NfsClientGroupRule{} + //r2 := &NfsClientGroupRule{} + + req1 := NewNfsClientGroupRuleCreateRequest(cg.Uid, NfsClientGroupRuleTypeIP, "192.168.1.1") + req2 := NewNfsClientGroupRuleCreateRequest(cg.Uid, NfsClientGroupRuleTypeIP, "192.168.2.0/24") + req3 := NewNfsClientGroupRuleCreateRequest(cg.Uid, NfsClientGroupRuleTypeIP, "192.168.3.0/255.255.255.255") + req4 := NewNfsClientGroupRuleCreateRequest(cg.Uid, NfsClientGroupRuleTypeDNS, "test-hostname") +outerLoop: + for _, req := range []*NfsClientGroupRuleCreateRequest{req1, req2, req3, req4} { + for _, rule := range cg.Rules { + if rule.Type == req.Type && rule.Rule == req.AsRule() { + continue outerLoop + } + } + assert.NotNil(t, req) + //req2 := &NfsClientGroupRuleCreateRequest{Type: NfsClientGroupRuleTypeDNS, Hostname: "test-hostname", NfsClientGroupUid: cg.Uid} + + err = apiClient.CreateNfsClientGroupRule(context.Background(), req, r) + assert.NoError(t, err) + } + rules := &[]NfsClientGroupRule{} + err = apiClient.GetNfsClientGroupRules(context.Background(), NfsClientGroupName, rules) + assert.NoError(t, err) + assert.NotEmpty(t, rules) + for _, rule := range *rules { + assert.NotEmpty(t, rule.Uid) + assert.NotEmpty(t, rule.Type) + assert.NotEmpty(t, rule.Rule) + assert.NotEmpty(t, rule.Id) + } +} + +func TestNfsEnsureNfsPermissions(t *testing.T) { + apiClient := GetApiClientForTest(t) + + // Test EnsureNfsPermission + err := apiClient.EnsureNfsPermissions(context.Background(), "172.16.5.147", "default", NfsVersionV4, NfsClientGroupName) + assert.NoError(t, err) +} + +func TestInterfaceGroup(t *testing.T) { + apiClient := GetApiClientForTest(t) + + var igs []InterfaceGroup + var ig = &InterfaceGroup{ + Uid: uuid.New(), + } + // Test GetApiUrl + assert.NotEmpty(t, ig.GetApiUrl(apiClient)) + assert.Contains(t, ig.GetApiUrl(apiClient), ig.Uid.String()) + + // Test EQ + ig1 := &InterfaceGroup{ + Name: "test", + } + + ig2 := &InterfaceGroup{ + Name: "test", + } + assert.True(t, ig1.EQ(ig2)) + + // Test GetBasePath + assert.NotEmpty(t, ig.GetBasePath(apiClient)) + + // Test Create + // Test GetGroups + err := apiClient.GetInterfaceGroups(context.Background(), &igs) + assert.NoError(t, err) + assert.NotEmpty(t, igs) + assert.NotEmpty(t, igs[0].Ips) + // + //// Test GetGroupByUid + //uid := ig.Uid + //err = apiClient.GetInterfaceGroupByUid(context.Background(), uid, ig) + //assert.NoError(t, err) + //assert.Equal(t, igName, ig.Name) + //assert.NotEmpty(t, ig.Uid) + // + //// Test GetGroupByName + //name := ig.Name + //ig, err = apiClient.GetInterfaceGroupByName(context.Background(), name) + //assert.NoError(t, err) + //assert.Equal(t, igName, ig.Name) + //assert.NotEmpty(t, ig.Uid) + // + //// Test Delete + //r := &InterfaceGroupDeleteRequest{Uid: ig.Uid} + //err = apiClient.DeleteInterfaceGroup(context.Background(), r) + //assert.NoError(t, err) + //err = apiClient.GetInterfaceGroups(context.Background(), &igs) + //assert.NoError(t, err) + //for _, r := range igs { + // if r.Uid == ig.Uid { + // t.Errorf("Failed to delete group") + // } + //} +} + +func TestIsSupersetOf(t *testing.T) { + // Test case 1: IP rule superset + rule1 := &NfsClientGroupRule{ + Type: NfsClientGroupRuleTypeIP, + Rule: "192.168.1.0/24", + } + rule2 := &NfsClientGroupRule{ + Type: NfsClientGroupRuleTypeIP, + Rule: "192.168.1.1", + } + assert.True(t, rule1.IsSupersetOf(rule2)) + + // Test case 2: IP rule not superset + rule3 := &NfsClientGroupRule{ + Type: NfsClientGroupRuleTypeIP, + Rule: "192.168.2.0/24", + } + assert.False(t, rule1.IsSupersetOf(rule3)) + + // Test case 3: Non-IP rule + rule4 := &NfsClientGroupRule{ + Type: NfsClientGroupRuleTypeDNS, + Rule: "example.com", + } + assert.False(t, rule1.IsSupersetOf(rule4)) + + // Test case 4: Same rule + assert.True(t, rule1.IsSupersetOf(rule1)) +} diff --git a/pkg/wekafs/apiclient/utils.go b/pkg/wekafs/apiclient/utils.go index 498c49e6e..9eede7da8 100644 --- a/pkg/wekafs/apiclient/utils.go +++ b/pkg/wekafs/apiclient/utils.go @@ -1,8 +1,16 @@ package apiclient import ( + "encoding/binary" + "errors" + "fmt" "github.com/rs/zerolog/log" + "hash/fnv" + "net" + "os" "reflect" + "strings" + "time" ) // ObjectsAreEqual returns true if both ApiObject have same immutable fields (other fields and nil fields are disregarded) @@ -39,3 +47,134 @@ func ObjectRequestHasRequiredFields(o ApiObjectRequest) bool { } return true } + +// hashString is a simple hash function that takes a string and returns a hash value in the range [0, n) +func hashString(s string, n int) int { + if n == 0 { + return 0 + } + + // Create a new FNV-1a hash + h := fnv.New32a() + + // Write the string to the hash + _, _ = h.Write([]byte(s)) + + // Get the hash sum as a uint32 + hashValue := h.Sum32() + + // Return the hash value in the range of [0, n) + return int(hashValue % uint32(n)) +} + +type Network struct { + IP net.IP + Subnet *net.IP +} + +func (n *Network) AsNfsRule() string { + return fmt.Sprintf("%s/%s", n.IP.String(), n.Subnet.String()) +} + +func (n *Network) GetMaskBits() int { + ip := n.Subnet.To4() + if ip == nil { + return 0 + } + // Count the number of 1 bits + mask := binary.BigEndian.Uint32(ip) + + // Count the number of set bits + cidrBits := 0 + for mask != 0 { + cidrBits += int(mask & 1) + mask >>= 1 + } + + return cidrBits +} + +func parseNetworkString(s string) (*Network, error) { + var ip, subnet net.IP + if strings.Contains(s, "/") { + parts := strings.Split(s, "/") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid CIDR notation: %s", s) + } + ip = net.ParseIP(parts[0]) + subnet = net.ParseIP(parts[1]) + if subnet == nil { + _, ipNet, err := net.ParseCIDR(s) + if err != nil { + return nil, fmt.Errorf("invalid CIDR notation: %s", s) + } + subnet = net.IP(ipNet.Mask) + } + } else { + ip = net.ParseIP(s) + subnet = net.ParseIP("255.255.255.255") + } + + return &Network{ + IP: ip, + Subnet: &subnet, + }, nil +} + +func (n *Network) ContainsIPAddress(ipStr string) bool { + ip := net.ParseIP(ipStr) + if ip == nil { + return false + } + + _, ipNet, err := net.ParseCIDR(fmt.Sprintf("%s/%s", n.IP.String(), n.Subnet.String())) + if err != nil || ipNet == nil { + _, ipNet, err = net.ParseCIDR(fmt.Sprintf("%s/%d", n.IP.String(), n.GetMaskBits())) + if err != nil || ipNet == nil { + return false + } + } + return ipNet.Contains(ip) +} + +func GetNodeIpAddress() string { + ret := os.Getenv("KUBE_NODE_IP_ADDRESS") + if ret != "" { + return ret + } + return "127.0.0.1" +} + +func GetNodeIpAddressByRouting(targetHost string) (string, error) { + rAddr, err := net.ResolveUDPAddr("udp", targetHost+":80") + if err != nil { + return "", err + } + + // Create a UDP connection to the resolved IP address + conn, err := net.DialUDP("udp", nil, rAddr) + if err != nil { + return "", err + } + defer conn.Close() + + // Set a deadline for the connection + err = conn.SetDeadline(time.Now().Add(1 * time.Second)) + if err != nil { + return "", err + } + + // Get the local address from the UDP connection + localAddr := conn.LocalAddr() + if localAddr == nil { + return "", errors.New("failed to get local address") + } + + // Extract the IP address from the local address + localIP, _, err := net.SplitHostPort(localAddr.String()) + if err != nil { + return "", err + } + + return localIP, nil +} diff --git a/pkg/wekafs/apiclient/utils_test.go b/pkg/wekafs/apiclient/utils_test.go new file mode 100644 index 000000000..d15814dd9 --- /dev/null +++ b/pkg/wekafs/apiclient/utils_test.go @@ -0,0 +1,49 @@ +package apiclient + +import ( + "github.com/rs/zerolog/log" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHashString(t *testing.T) { + testCases := []struct { + input string + n int + expected int + }{ + {"test", 10, 5}, + {"example", 10, 9}, + {"hash", 10, 1}, + {"string", 10, 8}, + {"", 10, 1}, + {"osi415-zbjgk-worker-0-t6g55", 10, 5}, + } + + for _, tc := range testCases { + t.Run(tc.input, func(t *testing.T) { + result := hashString(tc.input, tc.n) + assert.Equal(t, tc.expected, result) + }) + } +} + +func TestGetNodeIpAddressByRouting(t *testing.T) { + testCases := []struct { + targetHost string + }{ + {"8.8.8.8"}, + {"1.1.1.1"}, + {"localhost"}, + } + + for _, tc := range testCases { + t.Run(tc.targetHost, func(t *testing.T) { + ip, err := GetNodeIpAddressByRouting(tc.targetHost) + assert.NoError(t, err) + assert.NotEmpty(t, ip) + log.Info().Str("ip", ip).Msg("Node IP address") + }) + } +} diff --git a/pkg/wekafs/controllerserver.go b/pkg/wekafs/controllerserver.go index 3b71ef0a4..dd14a9094 100644 --- a/pkg/wekafs/controllerserver.go +++ b/pkg/wekafs/controllerserver.go @@ -44,7 +44,7 @@ type ControllerServer struct { csi.UnimplementedControllerServer caps []*csi.ControllerServiceCapability nodeID string - mounter *wekaMounter + mounter AnyMounter api *ApiStore config *DriverConfig semaphores map[string]*semaphore.Weighted @@ -67,7 +67,7 @@ func (cs *ControllerServer) getConfig() *DriverConfig { return cs.config } -func (cs *ControllerServer) getMounter() *wekaMounter { +func (cs *ControllerServer) getMounter() AnyMounter { return cs.mounter } @@ -105,7 +105,7 @@ func (cs *ControllerServer) ControllerModifyVolume(context.Context, *csi.Control panic("implement me") } -func NewControllerServer(nodeID string, api *ApiStore, mounter *wekaMounter, config *DriverConfig) *ControllerServer { +func NewControllerServer(nodeID string, api *ApiStore, mounter AnyMounter, config *DriverConfig) *ControllerServer { exposedCapabilities := []csi.ControllerServiceCapability_RPC_Type{ csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, diff --git a/pkg/wekafs/driverconfig.go b/pkg/wekafs/driverconfig.go index 3f3b0611d..6e9541966 100644 --- a/pkg/wekafs/driverconfig.go +++ b/pkg/wekafs/driverconfig.go @@ -33,6 +33,11 @@ type DriverConfig struct { maxConcurrencyPerOp map[string]int64 grpcRequestTimeout time.Duration allowProtocolContainers bool + allowNfsFailback bool + useNfs bool + interfaceGroupName string + clientGroupName string + nfsProtocolVersion string } func (dc *DriverConfig) Log() { @@ -41,7 +46,22 @@ func (dc *DriverConfig) Log() { Bool("allow_auto_fs_creation", dc.allowAutoFsCreation).Bool("allow_auto_fs_expansion", dc.allowAutoFsExpansion). Bool("advertise_snapshot_support", dc.advertiseSnapshotSupport).Bool("advertise_volume_clone_support", dc.advertiseVolumeCloneSupport). Bool("allow_insecure_https", dc.allowInsecureHttps).Bool("always_allow_snapshot_volumes", dc.alwaysAllowSnapshotVolumes). - Interface("mutually_exclusive_mount_options", dc.mutuallyExclusiveOptions).Msg("Starting driver with the following configuration") + Interface("mutually_exclusive_mount_options", dc.mutuallyExclusiveOptions). + Int64("max_create_volume_reqs", dc.maxConcurrencyPerOp["CreateVolume"]). + Int64("max_delete_volume_reqs", dc.maxConcurrencyPerOp["DeleteVolume"]). + Int64("max_expand_volume_reqs", dc.maxConcurrencyPerOp["ExpandVolume"]). + Int64("max_create_snapshot_reqs", dc.maxConcurrencyPerOp["CreateSnapshot"]). + Int64("max_delete_snapshot_reqs", dc.maxConcurrencyPerOp["DeleteSnapshot"]). + Int64("max_node_publish_volume_reqs", dc.maxConcurrencyPerOp["NodePublishVolume"]). + Int64("max_node_unpublish_volume_reqs", dc.maxConcurrencyPerOp["NodeUnpublishVolume"]). + Int("grpc_request_timeout_seconds", int(dc.grpcRequestTimeout.Seconds())). + Bool("allow_protocol_containers", dc.allowProtocolContainers). + Bool("allow_nfs_failback", dc.allowNfsFailback). + Bool("use_nfs", dc.useNfs). + Str("interface_group_name", dc.interfaceGroupName). + Str("client_group_name", dc.clientGroupName). + Msg("Starting driver with the following configuration") + } func NewDriverConfig(dynamicVolPath, VolumePrefix, SnapshotPrefix, SeedSnapshotPrefix, debugPath string, allowAutoFsCreation, allowAutoFsExpansion, allowSnapshotsOfLegacyVolumes bool, @@ -50,6 +70,8 @@ func NewDriverConfig(dynamicVolPath, VolumePrefix, SnapshotPrefix, SeedSnapshotP maxCreateVolumeReqs, maxDeleteVolumeReqs, maxExpandVolumeReqs, maxCreateSnapshotReqs, maxDeleteSnapshotReqs, maxNodePublishVolumeReqs, maxNodeUnpublishVolumeReqs int64, grpcRequestTimeoutSeconds int, allowProtocolContainers bool, + allowNfsFailback, useNfs bool, + interfaceGroupName, clientGroupName, nfsProtocolVersion string, ) *DriverConfig { var MutuallyExclusiveMountOptions []mutuallyExclusiveMountOptionSet @@ -89,6 +111,11 @@ func NewDriverConfig(dynamicVolPath, VolumePrefix, SnapshotPrefix, SeedSnapshotP maxConcurrencyPerOp: concurrency, grpcRequestTimeout: grpcRequestTimeout, allowProtocolContainers: allowProtocolContainers, + allowNfsFailback: allowNfsFailback, + useNfs: useNfs, + interfaceGroupName: interfaceGroupName, + clientGroupName: clientGroupName, + nfsProtocolVersion: nfsProtocolVersion, } } diff --git a/pkg/wekafs/gc.go b/pkg/wekafs/gc.go index 304758f02..49c2de494 100644 --- a/pkg/wekafs/gc.go +++ b/pkg/wekafs/gc.go @@ -21,27 +21,27 @@ type innerPathVolGc struct { isRunning map[string]bool isDeferred map[string]bool sync.Mutex - mounter *wekaMounter + mounter AnyMounter } -func initInnerPathVolumeGc(mounter *wekaMounter) *innerPathVolGc { +func initInnerPathVolumeGc(mounter AnyMounter) *innerPathVolGc { gc := innerPathVolGc{mounter: mounter} gc.isRunning = make(map[string]bool) gc.isDeferred = make(map[string]bool) return &gc } -func (gc *innerPathVolGc) triggerGcVolume(ctx context.Context, volume *Volume) { +func (gc *innerPathVolGc) triggerGcVolume(ctx context.Context, volume *Volume) error { op := "triggerGcVolume" ctx, span := otel.Tracer(TracerName).Start(ctx, op) defer span.End() ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) logger := log.Ctx(ctx).With().Str("volume_id", volume.GetId()).Logger() logger.Info().Msg("Triggering garbage collection of volume") - gc.moveVolumeToTrash(ctx, volume) // always do it synchronously + return gc.moveVolumeToTrash(ctx, volume) } -func (gc *innerPathVolGc) moveVolumeToTrash(ctx context.Context, volume *Volume) { +func (gc *innerPathVolGc) moveVolumeToTrash(ctx context.Context, volume *Volume) error { op := "moveVolumeToTrash" ctx, span := otel.Tracer(TracerName).Start(ctx, op) defer span.End() @@ -54,7 +54,7 @@ func (gc *innerPathVolGc) moveVolumeToTrash(ctx context.Context, volume *Volume) defer unmount() if err != nil { logger.Error().Err(err).Msg("Failed to mount filesystem for GC processing") - return + return err } volumeTrashLoc := filepath.Join(path, garbagePath) if err := os.MkdirAll(volumeTrashLoc, DefaultVolumePermissions); err != nil { @@ -68,6 +68,7 @@ func (gc *innerPathVolGc) moveVolumeToTrash(ctx context.Context, volume *Volume) if err := os.Rename(fullPath, newPath); err != nil { logger.Error().Err(err).Str("full_path", fullPath). Str("volume_trash_location", volumeTrashLoc).Msg("Failed to move volume contents to volumeTrashLoc") + return err } // NOTE: there is a problem of directory leaks here. If the volume innerPath is deeper than /csi-volumes/vol-name, // e.g. if using statically provisioned volume, we move only the deepest directory @@ -77,6 +78,7 @@ func (gc *innerPathVolGc) moveVolumeToTrash(ctx context.Context, volume *Volume) // 2024-07-29: apparently seems this is not a real problem since static volumes are not deleted this way // and dynamic volumes are always created inside the /csi-volumes logger.Debug().Str("full_path", fullPath).Str("volume_trash_location", volumeTrashLoc).Msg("Volume contents moved to trash") + return nil } func (gc *innerPathVolGc) purgeLeftovers(ctx context.Context, fs string, apiClient *apiclient.ApiClient) { diff --git a/pkg/wekafs/identityserver.go b/pkg/wekafs/identityserver.go index 898347c80..132cf746d 100644 --- a/pkg/wekafs/identityserver.go +++ b/pkg/wekafs/identityserver.go @@ -79,10 +79,15 @@ func (ids *identityServer) getConfig() *DriverConfig { } func (ids *identityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { + logger := log.Ctx(ctx) isReady := ids.getConfig().isInDevMode() || isWekaInstalled() if !isReady { - logger := log.Ctx(ctx) - logger.Error().Msg("Weka driver not running on host, not ready to perform operations") + if ids.getConfig().useNfs || ids.getConfig().allowNfsFailback { + isReady = true + } + } + if !isReady { + logger.Error().Msg("Weka driver not running on host and NFS transport is not configured, not ready to perform operations") } return &csi.ProbeResponse{ Ready: &wrapperspb.BoolValue{ diff --git a/pkg/wekafs/interfaces.go b/pkg/wekafs/interfaces.go index 75af3f50f..86fb90fe3 100644 --- a/pkg/wekafs/interfaces.go +++ b/pkg/wekafs/interfaces.go @@ -1,10 +1,47 @@ package wekafs +import ( + "context" + "github.com/wekafs/csi-wekafs/pkg/wekafs/apiclient" + "time" +) + type AnyServer interface { - getMounter() *wekaMounter + getMounter() AnyMounter getApiStore() *ApiStore getConfig() *DriverConfig - isInDevMode() bool // TODO: Rename to isInDevMode + isInDevMode() bool getDefaultMountOptions() MountOptions getNodeId() string } + +type AnyMounter interface { + NewMount(fsName string, options MountOptions) AnyMount + mountWithOptions(ctx context.Context, fsName string, mountOptions MountOptions, apiClient *apiclient.ApiClient) (string, error, UnmountFunc) + Mount(ctx context.Context, fs string, apiClient *apiclient.ApiClient) (string, error, UnmountFunc) + unmountWithOptions(ctx context.Context, fsName string, options MountOptions) error + LogActiveMounts() + gcInactiveMounts() + schedulePeriodicMountGc() + getGarbageCollector() *innerPathVolGc +} + +type mountsMapPerFs map[string]AnyMount +type mountsMap map[string]mountsMapPerFs +type nfsMountsMap map[string]int // we only follow the mountPath and number of references + +type UnmountFunc func() + +type AnyMount interface { + isInDevMode() bool + isMounted() bool + incRef(ctx context.Context, apiClient *apiclient.ApiClient) error + decRef(ctx context.Context) error + getRefCount() int + doUnmount(ctx context.Context) error + doMount(ctx context.Context, apiClient *apiclient.ApiClient, mountOptions MountOptions) error + getMountPoint() string + getMountOptions() MountOptions + getLastUsed() time.Time + locateMountIP() error // used only for NFS +} diff --git a/pkg/wekafs/mountoptions.go b/pkg/wekafs/mountoptions.go index f161a0e7b..9e7f3568f 100644 --- a/pkg/wekafs/mountoptions.go +++ b/pkg/wekafs/mountoptions.go @@ -8,12 +8,15 @@ import ( ) const ( - selinuxContext = "wekafs_csi_volume" + selinuxContextWekaFs = "wekafs_csi_volume_t" + selinuxContextNfs = "nfs_t" MountOptionSyncOnClose = "sync_on_close" MountOptionReadOnly = "ro" MountOptionWriteCache = "writecache" MountOptionCoherent = "coherent" MountOptionReadCache = "readcache" + MountProtocolWekafs = "wekafs" + MountProtocolNfs = "nfs" ) type mountOption struct { @@ -145,15 +148,67 @@ func (opts MountOptions) Hash() uint32 { return h.Sum32() } -func (opts MountOptions) setSelinux(selinuxSupport bool) { +func (opts MountOptions) AsMapKey() string { + ret := opts + // TODO: if adding any other version-agnostic options, add them here + excludedOpts := []string{MountOptionSyncOnClose} + for _, o := range excludedOpts { + ret = ret.RemoveOption(o) + } + return ret.String() +} + +func (opts MountOptions) setSelinux(selinuxSupport bool, mountProtocol string) { if selinuxSupport { - o := newMountOptionFromString(fmt.Sprintf("fscontext=\"system_u:object_r:%s_t:s0\"", selinuxContext)) + var o mountOption + if mountProtocol == MountProtocolWekafs { + o = newMountOptionFromString(fmt.Sprintf("fscontext=\"system_u:object_r:%s:s0\"", selinuxContextWekaFs)) + } else if mountProtocol == MountProtocolNfs { + o = newMountOptionFromString(fmt.Sprintf("context=\"system_u:object_r:%s:s0\"", selinuxContextNfs)) + } opts.customOptions[o.option] = o } else { - delete(opts.customOptions, "fscontext") + if mountProtocol == MountProtocolWekafs { + delete(opts.customOptions, "fscontext") + } + if mountProtocol == MountProtocolNfs { + delete(opts.customOptions, "context") + } } } +func (opts MountOptions) AsNfs() MountOptions { + ret := NewMountOptionsFromString("hard,rdirplus") + for _, o := range opts.getOpts() { + switch o.option { + case "writecache": + ret.AddOption("async") + case "coherent": + ret.AddOption("sync") + case "forcedirect": + ret.AddOption("sync") + case "readcache": + ret.AddOption("noac") + case "dentry_max_age_positive": + ret.AddOption(fmt.Sprintf("acdirmax=%s", o.value)) + ret.AddOption(fmt.Sprintf("acregmax=%s", o.value)) + case "inode_bits": + continue + case "verbose": + continue + case "quiet": + continue + case "obs_direct": + continue + case "sync_on_close": + ret.AddOption("sync") + default: + continue + } + } + return ret +} + func NewMountOptionsFromString(optsString string) MountOptions { if optsString == "" { return NewMountOptions([]string{}) diff --git a/pkg/wekafs/nfsmount.go b/pkg/wekafs/nfsmount.go new file mode 100644 index 000000000..13a385ade --- /dev/null +++ b/pkg/wekafs/nfsmount.go @@ -0,0 +1,213 @@ +package wekafs + +import ( + "context" + "errors" + "fmt" + "github.com/rs/zerolog/log" + "github.com/wekafs/csi-wekafs/pkg/wekafs/apiclient" + "k8s.io/mount-utils" + "os" + "path/filepath" + "strings" + "time" +) + +type nfsMount struct { + mounter *nfsMounter + fsName string + mountPoint string + kMounter mount.Interface + debugPath string + mountOptions MountOptions + lastUsed time.Time + mountIpAddress string + interfaceGroupName *string + clientGroupName string + protocolVersion apiclient.NfsVersionString +} + +func (m *nfsMount) getMountPoint() string { + return fmt.Sprintf("%s-%s", m.mountPoint, m.mountIpAddress) +} + +func (m *nfsMount) getRefCount() int { + return 0 +} + +func (m *nfsMount) getMountOptions() MountOptions { + return m.mountOptions.AddOption(fmt.Sprintf("vers=%s", m.protocolVersion.AsOption())) +} + +func (m *nfsMount) getLastUsed() time.Time { + return m.lastUsed +} + +func (m *nfsMount) isInDevMode() bool { + return m.debugPath != "" +} + +func (m *nfsMount) isMounted() bool { + return PathExists(m.getMountPoint()) && PathIsWekaMount(context.Background(), m.mountPoint) +} + +func (m *nfsMount) incRef(ctx context.Context, apiClient *apiclient.ApiClient) error { + logger := log.Ctx(ctx) + if m.mounter == nil { + logger.Error().Msg("Mounter is nil") + return errors.New("mounter is nil") + } + m.mounter.lock.Lock() + defer m.mounter.lock.Unlock() + refCount, ok := m.mounter.mountMap[m.getMountPoint()] + if !ok { + refCount = 0 + } + if refCount == 0 { + if err := m.doMount(ctx, apiClient, m.getMountOptions()); err != nil { + return err + } + } else if !m.isMounted() { + logger.Warn().Str("mount_point", m.getMountPoint()).Int("refcount", refCount).Msg("Mount not exists although should!") + if err := m.doMount(ctx, apiClient, m.getMountOptions()); err != nil { + return err + } + + } + refCount++ + m.mounter.mountMap[m.getMountPoint()] = refCount + + logger.Trace().Int("refcount", refCount).Strs("mount_options", m.getMountOptions().Strings()).Str("filesystem_name", m.fsName).Msg("RefCount increased") + return nil +} + +func (m *nfsMount) decRef(ctx context.Context) error { + logger := log.Ctx(ctx) + if m.mounter == nil { + logger.Error().Msg("Mounter is nil") + return errors.New("mounter is nil") + } + m.mounter.lock.Lock() + defer m.mounter.lock.Unlock() + refCount, ok := m.mounter.mountMap[m.getMountPoint()] + defer func() { + if refCount == 0 { + delete(m.mounter.mountMap, m.getMountPoint()) + } else { + m.mounter.mountMap[m.getMountPoint()] = refCount + } + }() + if !ok { + refCount = 0 + } + if refCount < 0 { + logger.Error().Int("refcount", refCount).Msg("During decRef negative refcount encountered") + refCount = 0 // to make sure that we don't have negative refcount later + } + if refCount == 1 { + if err := m.doUnmount(ctx); err != nil { + return err + } + refCount-- + } + return nil +} + +func (m *nfsMount) locateMountIP() error { + if m.mountIpAddress == "" { + ipAddr, err := GetMountIpFromActualMountPoint(m.mountPoint) + if err != nil { + return err + } + m.mountIpAddress = ipAddr + } + return nil +} + +func (m *nfsMount) doUnmount(ctx context.Context) error { + logger := log.Ctx(ctx).With().Str("mount_point", m.getMountPoint()).Str("filesystem", m.fsName).Logger() + logger.Trace().Strs("mount_options", m.getMountOptions().Strings()).Msg("Performing umount via k8s native mounter") + err := m.kMounter.Unmount(m.getMountPoint()) + if err != nil { + logger.Error().Err(err).Msg("Failed to unmount") + } else { + logger.Trace().Msg("Unmounted successfully") + } + return err +} + +func (m *nfsMount) ensureMountIpAddress(ctx context.Context, apiClient *apiclient.ApiClient) error { + if m.mountIpAddress == "" { + ip, err := apiClient.GetNfsMountIp(ctx, m.interfaceGroupName) + if err != nil { + return err + } + m.mountIpAddress = ip + } + return nil +} + +func (m *nfsMount) doMount(ctx context.Context, apiClient *apiclient.ApiClient, mountOptions MountOptions) error { + logger := log.Ctx(ctx).With().Str("mount_point", m.getMountPoint()).Str("filesystem", m.fsName).Logger() + var mountOptionsSensitive []string + if apiClient == nil { + // this flow is relevant only for legacy volumes, will not work with SCMC + logger.Trace().Msg("No API client for mount, cannot proceed") + return errors.New("no API client for mount, cannot do NFS mount") + } + + if err := m.ensureMountIpAddress(ctx, apiClient); err != nil { + logger.Error().Err(err).Msg("Failed to get mount IP address") + return err + } + + if err := os.MkdirAll(m.getMountPoint(), DefaultVolumePermissions); err != nil { + return err + } + if !m.isInDevMode() { + + nodeIP, err := apiclient.GetNodeIpAddressByRouting(m.mountIpAddress) + if err != nil { + logger.Error().Err(err).Msg("Failed to get routed node IP address, relying on node IP") + nodeIP = apiclient.GetNodeIpAddress() + } + + if apiClient.EnsureNfsPermissions(ctx, nodeIP, m.fsName, apiclient.NfsVersionV4, m.clientGroupName) != nil { + logger.Error().Msg("Failed to ensure NFS permissions") + return errors.New("failed to ensure NFS permissions") + } + + mountTarget := m.mountIpAddress + ":/" + m.fsName + logger.Trace(). + Strs("mount_options", m.getMountOptions().Strings()). + Str("mount_target", mountTarget). + Str("mount_ip_address", m.mountIpAddress). + Msg("Performing mount") + + err = m.kMounter.MountSensitive(mountTarget, m.getMountPoint(), "nfs", mountOptions.Strings(), mountOptionsSensitive) + if err != nil { + if os.IsNotExist(err) { + logger.Error().Err(err).Msg("Mount target not found") + } else if os.IsPermission(err) { + logger.Error().Err(err).Msg("Mount failed due to permissions issue") + return err + } else if strings.Contains(err.Error(), "invalid argument") { + logger.Error().Err(err).Msg("Mount failed due to invalid argument") + return err + } else { + logger.Error().Err(err).Msg("Mount failed due to unknown issue") + } + return err + } + logger.Trace().Msg("Mounted successfully") + return nil + } else { + fakePath := filepath.Join(m.debugPath, m.fsName) + if err := os.MkdirAll(fakePath, DefaultVolumePermissions); err != nil { + Die(fmt.Sprintf("Failed to create directory %s, while running in debug mode", fakePath)) + } + logger.Trace().Strs("mount_options", m.getMountOptions().Strings()).Str("debug_path", m.debugPath).Msg("Performing mount") + + return m.kMounter.Mount(fakePath, m.getMountPoint(), "", []string{"bind"}) + } +} diff --git a/pkg/wekafs/nfsmounter.go b/pkg/wekafs/nfsmounter.go new file mode 100644 index 000000000..00e0381eb --- /dev/null +++ b/pkg/wekafs/nfsmounter.go @@ -0,0 +1,164 @@ +package wekafs + +import ( + "context" + "fmt" + "github.com/rs/zerolog/log" + "github.com/wekafs/csi-wekafs/pkg/wekafs/apiclient" + "k8s.io/mount-utils" + "sync" + "time" +) + +type nfsMounter struct { + mountMap nfsMountsMap + lock sync.Mutex + kMounter mount.Interface + debugPath string + selinuxSupport *bool + gc *innerPathVolGc + interfaceGroupName *string + clientGroupName string + nfsProtocolVersion string + exclusiveMountOptions []mutuallyExclusiveMountOptionSet +} + +func (m *nfsMounter) getGarbageCollector() *innerPathVolGc { + return m.gc +} + +func newNfsMounter(driver *WekaFsDriver) *nfsMounter { + var selinuxSupport *bool + if driver.selinuxSupport { + log.Debug().Msg("SELinux support is forced") + selinuxSupport = &[]bool{true}[0] + } + mounter := &nfsMounter{mountMap: make(nfsMountsMap), debugPath: driver.debugPath, selinuxSupport: selinuxSupport, exclusiveMountOptions: driver.config.mutuallyExclusiveOptions} + mounter.gc = initInnerPathVolumeGc(mounter) + mounter.schedulePeriodicMountGc() + mounter.interfaceGroupName = &driver.config.interfaceGroupName + mounter.clientGroupName = driver.config.clientGroupName + mounter.nfsProtocolVersion = driver.config.nfsProtocolVersion + + return mounter +} + +func (m *nfsMounter) NewMount(fsName string, options MountOptions) AnyMount { + if m.kMounter == nil { + m.kMounter = mount.New("") + } + uniqueId := getStringSha1AsB32(fsName + ":" + options.String()) + wMount := &nfsMount{ + mounter: m, + kMounter: m.kMounter, + fsName: fsName, + debugPath: m.debugPath, + mountPoint: "/run/weka-fs-mounts/" + getAsciiPart(fsName, 64) + "-" + uniqueId, + mountOptions: options, + interfaceGroupName: m.interfaceGroupName, + clientGroupName: m.clientGroupName, + protocolVersion: apiclient.NfsVersionString(fmt.Sprintf("V%s", m.nfsProtocolVersion)), + } + return wMount +} + +func (m *nfsMounter) getSelinuxStatus(ctx context.Context) bool { + if m.selinuxSupport != nil && *m.selinuxSupport { + return true + } + selinuxSupport := getSelinuxStatus(ctx) + m.selinuxSupport = &selinuxSupport + return *m.selinuxSupport +} + +func (m *nfsMounter) mountWithOptions(ctx context.Context, fsName string, mountOptions MountOptions, apiClient *apiclient.ApiClient) (string, error, UnmountFunc) { + mountOptions.setSelinux(m.getSelinuxStatus(ctx), MountProtocolNfs) + mountOptions = mountOptions.AsNfs() + mountOptions.Merge(mountOptions, m.exclusiveMountOptions) + mountObj := m.NewMount(fsName, mountOptions) + mountErr := mountObj.incRef(ctx, apiClient) + + if mountErr != nil { + log.Ctx(ctx).Error().Err(mountErr).Msg("Failed mounting") + return "", mountErr, func() {} + } + return mountObj.getMountPoint(), nil, func() { + if mountErr == nil { + _ = mountObj.decRef(ctx) + } + } +} + +func (m *nfsMounter) Mount(ctx context.Context, fs string, apiClient *apiclient.ApiClient) (string, error, UnmountFunc) { + return m.mountWithOptions(ctx, fs, getDefaultMountOptions(), apiClient) +} + +func (m *nfsMounter) unmountWithOptions(ctx context.Context, fsName string, options MountOptions) error { + opts := options + options.setSelinux(m.getSelinuxStatus(ctx), MountProtocolNfs) + options = options.AsNfs() + options.Merge(options, m.exclusiveMountOptions) + mnt := m.NewMount(fsName, options) + // since we are not aware of the IP address of the mount, we need to find the mount point by listing the mounts + err := mnt.locateMountIP() + if err != nil { + log.Ctx(ctx).Error().Err(err).Msg("Failed to locate mount IP") + return err + } + + log.Ctx(ctx).Trace().Strs("mount_options", opts.Strings()).Str("filesystem", fsName).Msg("Received an unmount request") + return mnt.decRef(ctx) +} + +func (m *nfsMounter) LogActiveMounts() { + //if len(m.mountMap) > 0 { + // count := 0 + // for fsName := range m.mountMap { + // for mnt := range m.mountMap[fsName] { + // mapEntry := m.mountMap[fsName][mnt] + // if mapEntry.getRefCount() > 0 { + // log.Trace().Str("filesystem", fsName).Int("refcount", mapEntry.getRefCount()).Strs("mount_options", mapEntry.getMountOptions().Strings()).Msg("Mount is active") + // count++ + // } else { + // log.Trace().Str("filesystem", fsName).Int("refcount", mapEntry.getRefCount()).Strs("mount_options", mapEntry.getMountOptions().Strings()).Msg("Mount is not active") + // } + // + // } + // } + // log.Debug().Int("total", len(m.mountMap)).Int("active", count).Msg("Periodic checkup on mount map") + //} +} + +func (m *nfsMounter) gcInactiveMounts() { + //if len(m.mountMap) > 0 { + // for fsName := range m.mountMap { + // for uniqueId, wekaMount := range m.mountMap[fsName] { + // if wekaMount.getRefCount() == 0 { + // if wekaMount.getLastUsed().Before(time.Now().Add(-inactiveMountGcPeriod)) { + // m.lock.Lock() + // if wekaMount.getRefCount() == 0 { + // log.Trace().Str("filesystem", fsName).Strs("mount_options", wekaMount.getMountOptions().Strings()). + // Time("last_used", wekaMount.getLastUsed()).Msg("Removing stale mount from map") + // delete(m.mountMap[fsName], uniqueId) + // } + // m.lock.Unlock() + // } + // } + // } + // if len(m.mountMap[fsName]) == 0 { + // delete(m.mountMap, fsName) + // } + // } + //} +} + +func (m *nfsMounter) schedulePeriodicMountGc() { + go func() { + log.Debug().Msg("Initializing periodic mount GC") + for true { + m.LogActiveMounts() + m.gcInactiveMounts() + time.Sleep(10 * time.Minute) + } + }() +} diff --git a/pkg/wekafs/nodeserver.go b/pkg/wekafs/nodeserver.go index 630a1ade0..c8ec97065 100644 --- a/pkg/wekafs/nodeserver.go +++ b/pkg/wekafs/nodeserver.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/pkg/errors" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "go.opentelemetry.io/otel" @@ -32,6 +33,7 @@ import ( "path/filepath" "strings" "sync" + "syscall" "time" ) @@ -48,7 +50,7 @@ type NodeServer struct { caps []*csi.NodeServiceCapability nodeID string maxVolumesPerNode int64 - mounter *wekaMounter + mounter AnyMounter api *ApiStore config *DriverConfig semaphores map[string]*semaphore.Weighted @@ -75,7 +77,7 @@ func (ns *NodeServer) getApiStore() *ApiStore { return ns.api } -func (ns *NodeServer) getMounter() *wekaMounter { +func (ns *NodeServer) getMounter() AnyMounter { return ns.mounter } @@ -84,16 +86,104 @@ func (ns *NodeServer) NodeExpandVolume(ctx context.Context, request *csi.NodeExp panic("implement me") } -//goland:noinspection GoUnusedParameter -func (ns *NodeServer) NodeGetVolumeStats(ctx context.Context, request *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { - panic("implement me") +func (ns *NodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { + volumeID := req.GetVolumeId() + volumePath := req.GetVolumePath() + + // Validate request fields + if volumeID == "" || volumePath == "" { + return nil, status.Error(codes.InvalidArgument, "Volume ID and path must be provided") + } + + // Check if the volume path exists + if ns.getConfig().isInDevMode() { + // In dev mode, we don't have the actual Weka mount, so we just check if the path exists + if _, err := os.Stat(volumePath); err != nil { + return nil, status.Error(codes.NotFound, "Volume path not found") + } + + } else { + // In production mode, we check if the path is indeed a Weka mount (Either NFS or WekaFS) + if !PathIsWekaMount(ctx, volumePath) { + return nil, status.Error(codes.NotFound, "Volume path not found") + } + } + + // Validate Weka volume ID + if err := validateVolumeId(volumeID); err != nil { + return nil, status.Error(codes.InvalidArgument, errors.Wrap(err, "invalid volume ID").Error()) + } + + stats, err := getVolumeStats(volumePath) + if err != nil || stats == nil { + return &csi.NodeGetVolumeStatsResponse{ + Usage: nil, + VolumeCondition: &csi.VolumeCondition{ + Abnormal: true, + Message: "Failed to fetch volume stats for volume", + }, + }, status.Errorf(codes.Internal, "Failed to get stats for volume %s: %v", volumeID, err) + } + // Prepare response + return &csi.NodeGetVolumeStatsResponse{ + Usage: []*csi.VolumeUsage{ + { + Unit: csi.VolumeUsage_BYTES, + Total: stats.TotalBytes, + Used: stats.UsedBytes, + Available: stats.AvailableBytes, + }, + { + Unit: csi.VolumeUsage_INODES, + Total: stats.TotalInodes, + Used: stats.UsedInodes, + Available: stats.AvailableInodes, + }, + }, + VolumeCondition: &csi.VolumeCondition{ + Abnormal: false, + Message: "volume is healthy", + }, + }, nil +} + +type VolumeStats struct { + TotalBytes int64 + UsedBytes int64 + AvailableBytes int64 + TotalInodes int64 + UsedInodes int64 + AvailableInodes int64 +} + +// getVolumeStats fetches filesystem statistics for the mounted volume path. +func getVolumeStats(volumePath string) (volumeStats *VolumeStats, err error) { + var stat syscall.Statfs_t + + // Use Statfs to get filesystem statistics for the volume path + err = syscall.Statfs(volumePath, &stat) + if err != nil { + return nil, err + } + + // Calculate capacity, available, and used space in bytes + capacityBytes := int64(stat.Blocks) * int64(stat.Bsize) + availableBytes := int64(stat.Bavail) * int64(stat.Bsize) + usedBytes := capacityBytes - availableBytes + inodes := int64(stat.Files) + inodesFree := int64(stat.Ffree) + inodesUsed := inodes - inodesFree + return &VolumeStats{capacityBytes, usedBytes, availableBytes, inodes, inodesUsed, inodesFree}, nil } -func NewNodeServer(nodeId string, maxVolumesPerNode int64, api *ApiStore, mounter *wekaMounter, config *DriverConfig) *NodeServer { +func NewNodeServer(nodeId string, maxVolumesPerNode int64, api *ApiStore, mounter AnyMounter, config *DriverConfig) *NodeServer { //goland:noinspection GoBoolExpressions return &NodeServer{ caps: getNodeServiceCapabilities( []csi.NodeServiceCapability_RPC_Type{ + csi.NodeServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER, + csi.NodeServiceCapability_RPC_GET_VOLUME_STATS, + csi.NodeServiceCapability_RPC_VOLUME_CONDITION, //csi.NodeServiceCapability_RPC_EXPAND_VOLUME, }, ), @@ -251,9 +341,11 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis err, unmount := volume.MountUnderlyingFS(ctx) if err != nil { - unmount() + logger.Error().Err(err).Msg("Failed to mount underlying filesystem") return NodePublishVolumeError(ctx, codes.Internal, "Failed to mount a parent filesystem, check Authentication: "+err.Error()) } + defer unmount() // unmount the parent mount since there is a bind mount anyway + fullPath := volume.GetFullPath(ctx) targetPathDir := filepath.Dir(targetPath) @@ -294,10 +386,8 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis // if we run in K8s isolated environment, 2nd mount must be done using mapped volume path if err := mounter.Mount(fullPath, targetPath, "", innerMountOpts); err != nil { - var errList strings.Builder - errList.WriteString(err.Error()) - unmount() // unmount only if mount bind failed - return NodePublishVolumeError(ctx, codes.Internal, fmt.Sprintf("failed to Mount device: %s at %s: %s", fullPath, targetPath, errList.String())) + logger.Error().Err(err).Str("full_path", fullPath).Str("target_path", targetPath).Msg("Failed to perform mount") + return NodePublishVolumeError(ctx, codes.Internal, fmt.Sprintf("failed to Mount device: %s at %s: %s", fullPath, targetPath, err.Error())) } result = "SUCCESS" // Not doing unmount, NodePublish should do unmount but only when it unmounts bind successfully @@ -314,7 +404,6 @@ func NodeUnpublishVolumeError(ctx context.Context, errorCode codes.Code, errorMe func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { op := "NodeUnpublishVolume" result := "FAILURE" - volumeID := req.GetVolumeId() ctx, span := otel.Tracer(TracerName).Start(ctx, op, trace.WithNewRoot()) defer span.End() ctx = log.With().Str("trace_id", span.SpanContext().TraceID().String()).Str("span_id", span.SpanContext().SpanID().String()).Str("op", op).Logger().WithContext(ctx) @@ -337,12 +426,6 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return NodeUnpublishVolumeError(ctx, codes.Unavailable, "Too many concurrent requests, please retry") } - // Check arguments - volume, err := NewVolumeFromId(ctx, req.GetVolumeId(), nil, ns) - if err != nil { - return &csi.NodeUnpublishVolumeResponse{}, err - } - if len(req.GetTargetPath()) == 0 { return NodeUnpublishVolumeError(ctx, codes.InvalidArgument, "Target path missing in request") } @@ -358,6 +441,7 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu result = "SUCCESS" return &csi.NodeUnpublishVolumeResponse{}, nil } else { + logger.Error().Err(err).Msg("Failed to check target path") return NodeUnpublishVolumeError(ctx, codes.Internal, "unexpected situation, please contact support") } @@ -386,11 +470,6 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return NodeUnpublishVolumeError(ctx, codes.Internal, err.Error()) } - logger.Trace().Str("volume_id", volumeID).Msg("Unmounting") - err = volume.UnmountUnderlyingFS(ctx) - if err != nil { - logger.Error().Str("volume_id", volumeID).Err(err).Msg("Post-unpublish task failed") - } result = "SUCCESS" return &csi.NodeUnpublishVolumeResponse{}, nil } diff --git a/pkg/wekafs/utilities.go b/pkg/wekafs/utilities.go index 7dd36f3ca..07b77d315 100644 --- a/pkg/wekafs/utilities.go +++ b/pkg/wekafs/utilities.go @@ -294,11 +294,33 @@ func PathIsWekaMount(ctx context.Context, path string) bool { if len(fields) >= 3 && fields[2] == "wekafs" && fields[1] == path { return true } + // TODO: better protect against false positives + if len(fields) >= 3 && strings.HasPrefix(fields[2], "nfs") && fields[1] == path { + return true + } } return false } +func GetMountIpFromActualMountPoint(mountPointBase string) (string, error) { + file, err := os.Open("/proc/mounts") + if err != nil { + return "", errors.New("failed to open /proc/mounts") + } + defer func() { _ = file.Close() }() + var actualMountPoint string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) >= 3 && strings.HasPrefix(fields[1], fmt.Sprintf("%s-", mountPointBase)) { + actualMountPoint = fields[1] + return strings.TrimLeft(actualMountPoint, mountPointBase+"-"), nil + } + } + return "", errors.New("mount point not found") +} + func validateVolumeId(volumeId string) error { // Volume New format: // VolID format is as following: @@ -509,3 +531,29 @@ func isWekaInstalled() bool { } return false } + +func getSelinuxStatus(ctx context.Context) bool { + logger := log.Ctx(ctx) + // check if we have /etc/selinux/config + // if it exists, we can check if selinux is enforced or not + selinuxConf := "/etc/selinux/config" + file, err := os.Open(selinuxConf) + if err != nil { + return false + } + defer func() { _ = file.Close() }() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, "SELINUX=enforcing") { + // no need to repeat each time, just set the selinuxSupport to true + return true + } + } + + if err := scanner.Err(); err != nil { + logger.Error().Err(err).Str("filename", selinuxConf).Msg("Failed to read SELinux config file") + } + return false +} diff --git a/pkg/wekafs/volume.go b/pkg/wekafs/volume.go index 44135f543..d807d52ab 100644 --- a/pkg/wekafs/volume.go +++ b/pkg/wekafs/volume.go @@ -88,9 +88,14 @@ func (v *Volume) initMountOptions(ctx context.Context) { func (v *Volume) pruneUnsupportedMountOptions(ctx context.Context) { logger := log.Ctx(ctx) - if v.mountOptions.hasOption(MountOptionSyncOnClose) && (v.apiClient == nil || !v.apiClient.SupportsSyncOnCloseMountOption()) { - logger.Debug().Str("mount_option", MountOptionSyncOnClose).Msg("Mount option not supported by current Weka cluster version and is dropped.") - v.mountOptions = v.mountOptions.RemoveOption(MountOptionSyncOnClose) + if v.mountOptions.hasOption(MountOptionSyncOnClose) { + if v.apiClient != nil && !v.apiClient.SupportsSyncOnCloseMountOption() { + logger.Debug().Str("mount_option", MountOptionSyncOnClose).Msg("Mount option not supported by current Weka cluster version and is dropped.") + v.mountOptions = v.mountOptions.RemoveOption(MountOptionSyncOnClose) + } else if v.apiClient == nil { + logger.Debug().Str("mount_option", MountOptionSyncOnClose).Msg("Cannot determine current Weka cluster version, dropping mount option.") + v.mountOptions = v.mountOptions.RemoveOption(MountOptionSyncOnClose) + } } if v.mountOptions.hasOption(MountOptionReadOnly) { logger.Error().Str("mount_option", MountOptionReadOnly).Msg("Mount option is not supported via custom mount options, use readOnly volume attachments instead") @@ -628,8 +633,7 @@ func (v *Volume) updateCapacityXattr(ctx context.Context, enforceCapacity *bool, func (v *Volume) Trash(ctx context.Context) error { if v.requiresGc() { - v.server.getMounter().gc.triggerGcVolume(ctx, v) - return nil + return v.server.getMounter().getGarbageCollector().triggerGcVolume(ctx, v) } return v.Delete(ctx) } diff --git a/pkg/wekafs/wekafs.go b/pkg/wekafs/wekafs.go index 01241bcf2..6f1d00123 100644 --- a/pkg/wekafs/wekafs.go +++ b/pkg/wekafs/wekafs.go @@ -110,6 +110,10 @@ func (api *ApiStore) fromSecrets(ctx context.Context, secrets map[string]string, if ok { autoUpdateEndpoints = strings.TrimSpace(strings.TrimSuffix(autoUpdateEndpointsStr, "\n")) == "true" } + caCertificate, ok := secrets["caCertificate"] + if !ok { + caCertificate = "" + } credentials := apiclient.Credentials{ Username: strings.TrimSpace(strings.TrimSuffix(secrets["username"], "\n")), @@ -119,6 +123,7 @@ func (api *ApiStore) fromSecrets(ctx context.Context, secrets map[string]string, HttpScheme: strings.TrimSpace(strings.TrimSuffix(secrets["scheme"], "\n")), LocalContainerName: localContainerName, AutoUpdateEndpoints: autoUpdateEndpoints, + CaCertificate: caCertificate, } return api.fromCredentials(ctx, credentials, hostname) } @@ -259,8 +264,9 @@ func NewWekaFsDriver( } func (driver *WekaFsDriver) Run() { + mounter := driver.NewMounter() + // Create GRPC servers - mounter := newWekaMounter(driver) // identity server runs always log.Info().Msg("Loading IdentityServer") @@ -317,3 +323,21 @@ func GetCsiPluginMode(mode *string) CsiPluginMode { return "" } } + +func (driver *WekaFsDriver) NewMounter() AnyMounter { + log.Info().Msg("Configuring Mounter") + if driver.config.useNfs { + log.Warn().Msg("Enforcing NFS transport due to configuration") + return newNfsMounter(driver) + } + if driver.config.allowNfsFailback && !isWekaInstalled() { + if driver.config.isInDevMode() { + log.Info().Msg("Not Enforcing NFS transport due to dev mode") + } else { + log.Warn().Msg("Weka Driver not found. Failing back to NFS transport") + return newNfsMounter(driver) + } + } + log.Info().Msg("Enforcing WekaFS transport") + return newWekafsMounter(driver) +} diff --git a/pkg/wekafs/mount.go b/pkg/wekafs/wekafsmount.go similarity index 57% rename from pkg/wekafs/mount.go rename to pkg/wekafs/wekafsmount.go index cf02f2d55..9e178c87c 100644 --- a/pkg/wekafs/mount.go +++ b/pkg/wekafs/wekafsmount.go @@ -13,7 +13,7 @@ import ( "time" ) -type wekaMount struct { +type wekafsMount struct { fsName string mountPoint string refCount int @@ -25,61 +25,77 @@ type wekaMount struct { allowProtocolContainers bool } -func (m *wekaMount) isInDevMode() bool { +func (m *wekafsMount) getMountPoint() string { + return m.mountPoint +} + +func (m *wekafsMount) getRefCount() int { + return m.refCount +} + +func (m *wekafsMount) getMountOptions() MountOptions { + return m.mountOptions +} +func (m *wekafsMount) getLastUsed() time.Time { + return m.lastUsed +} + +func (m *wekafsMount) isInDevMode() bool { return m.debugPath != "" } -func (m *wekaMount) isMounted() bool { - return PathExists(m.mountPoint) && PathIsWekaMount(context.Background(), m.mountPoint) +func (m *wekafsMount) isMounted() bool { + return PathExists(m.getMountPoint()) && PathIsWekaMount(context.Background(), m.getMountPoint()) } -func (m *wekaMount) incRef(ctx context.Context, apiClient *apiclient.ApiClient) error { +func (m *wekafsMount) incRef(ctx context.Context, apiClient *apiclient.ApiClient) error { logger := log.Ctx(ctx) m.lock.Lock() defer m.lock.Unlock() if m.refCount < 0 { - logger.Error().Str("mount_point", m.mountPoint).Int("refcount", m.refCount).Msg("During incRef negative refcount encountered") + logger.Error().Str("mount_point", m.getMountPoint()).Int("refcount", m.refCount).Msg("During incRef negative refcount encountered") m.refCount = 0 // to make sure that we don't have negative refcount later } if m.refCount == 0 { - if err := m.doMount(ctx, apiClient, m.mountOptions); err != nil { + if err := m.doMount(ctx, apiClient, m.getMountOptions()); err != nil { return err } } else if !m.isMounted() { - logger.Warn().Str("mount_point", m.mountPoint).Int("refcount", m.refCount).Msg("Mount not exists although should!") - if err := m.doMount(ctx, apiClient, m.mountOptions); err != nil { + logger.Warn().Str("mount_point", m.getMountPoint()).Int("refcount", m.refCount).Msg("Mount not exists although should!") + if err := m.doMount(ctx, apiClient, m.getMountOptions()); err != nil { return err } } m.refCount++ - logger.Trace().Int("refcount", m.refCount).Strs("mount_options", m.mountOptions.Strings()).Str("filesystem_name", m.fsName).Msg("RefCount increased") + logger.Trace().Int("refcount", m.refCount).Strs("mount_options", m.getMountOptions().Strings()).Str("filesystem_name", m.fsName).Msg("RefCount increased") return nil } -func (m *wekaMount) decRef(ctx context.Context) error { +func (m *wekafsMount) decRef(ctx context.Context) error { logger := log.Ctx(ctx) m.lock.Lock() defer m.lock.Unlock() m.refCount-- m.lastUsed = time.Now() - logger.Trace().Int("refcount", m.refCount).Strs("mount_options", m.mountOptions.Strings()).Str("filesystem_name", m.fsName).Msg("RefCount decreased") + logger.Trace().Int("refcount", m.refCount).Strs("mount_options", m.getMountOptions().Strings()).Str("filesystem_name", m.fsName).Msg("RefCount decreased") if m.refCount < 0 { logger.Error().Int("refcount", m.refCount).Msg("During decRef negative refcount encountered") m.refCount = 0 // to make sure that we don't have negative refcount later } if m.refCount == 0 { if err := m.doUnmount(ctx); err != nil { + m.refCount++ // since we failed to unmount, we need to increase the refcount back to the original value return err } } return nil } -func (m *wekaMount) doUnmount(ctx context.Context) error { - logger := log.Ctx(ctx).With().Str("mount_point", m.mountPoint).Str("filesystem", m.fsName).Logger() - logger.Trace().Strs("mount_options", m.mountOptions.Strings()).Msg("Performing umount via k8s native mounter") - err := m.kMounter.Unmount(m.mountPoint) +func (m *wekafsMount) doUnmount(ctx context.Context) error { + logger := log.Ctx(ctx).With().Str("mount_point", m.getMountPoint()).Str("filesystem", m.fsName).Logger() + logger.Trace().Strs("mount_options", m.getMountOptions().Strings()).Msg("Performing umount via k8s native mounter") + err := m.kMounter.Unmount(m.getMountPoint()) if err != nil { logger.Error().Err(err).Msg("Failed to unmount") } else { @@ -88,12 +104,12 @@ func (m *wekaMount) doUnmount(ctx context.Context) error { return err } -func (m *wekaMount) doMount(ctx context.Context, apiClient *apiclient.ApiClient, mountOptions MountOptions) error { - logger := log.Ctx(ctx).With().Str("mount_point", m.mountPoint).Str("filesystem", m.fsName).Logger() +func (m *wekafsMount) doMount(ctx context.Context, apiClient *apiclient.ApiClient, mountOptions MountOptions) error { + logger := log.Ctx(ctx).With().Str("mount_point", m.getMountPoint()).Str("filesystem", m.fsName).Logger() mountToken := "" var mountOptionsSensitive []string var localContainerName string - if err := os.MkdirAll(m.mountPoint, DefaultVolumePermissions); err != nil { + if err := os.MkdirAll(m.getMountPoint(), DefaultVolumePermissions); err != nil { return err } if !m.isInDevMode() { @@ -119,10 +135,10 @@ func (m *wekaMount) doMount(ctx context.Context, apiClient *apiclient.ApiClient, // if needed, add containerName to the mount string if apiClient != nil && len(containerPaths) > 1 { + localContainerName = apiClient.Credentials.LocalContainerName if apiClient.SupportsMultipleClusters() { - localContainerName = apiClient.Credentials.LocalContainerName if localContainerName != "" { - logger.Info().Str("local_container_name", localContainerName).Msg("Local container name set by secret") + logger.Info().Str("local_container_name", localContainerName).Msg("Local container name set by secrets") } else { container, err := apiClient.GetLocalContainer(ctx, m.allowProtocolContainers) if err != nil || container == nil { @@ -140,28 +156,31 @@ func (m *wekaMount) doMount(ctx context.Context, apiClient *apiclient.ApiClient, option: "container_name", value: localContainerName, } + break } } } else { - err = errors.New("mount failed, local container name not specified and could not be determined automatically, refer to documentation on handling multiple clusters clients with Kubernetes") - logger.Error().Err(err).Msg("Failed to mount") - return err + logger.Error().Err(errors.New("Could not determine container name, refer to documentation on handling multiple clusters clients with Kubernetes")).Msg("Failed to mount") } } } - logger.Trace().Strs("mount_options", m.mountOptions.Strings()). + logger.Trace().Strs("mount_options", m.getMountOptions().Strings()). Fields(mountOptions).Msg("Performing mount") - return m.kMounter.MountSensitive(m.fsName, m.mountPoint, "wekafs", mountOptions.Strings(), mountOptionsSensitive) + return m.kMounter.MountSensitive(m.fsName, m.getMountPoint(), "wekafs", mountOptions.Strings(), mountOptionsSensitive) } else { fakePath := filepath.Join(m.debugPath, m.fsName) if err := os.MkdirAll(fakePath, DefaultVolumePermissions); err != nil { Die(fmt.Sprintf("Failed to create directory %s, while running in debug mode", fakePath)) } - logger.Trace().Strs("mount_options", m.mountOptions.Strings()).Str("debug_path", m.debugPath).Msg("Performing mount") + logger.Trace().Strs("mount_options", m.getMountOptions().Strings()).Str("debug_path", m.debugPath).Msg("Performing mount") - return m.kMounter.Mount(fakePath, m.mountPoint, "", []string{"bind"}) + return m.kMounter.Mount(fakePath, m.getMountPoint(), "", []string{"bind"}) } } + +func (m *wekafsMount) locateMountIP() error { + return nil +} diff --git a/pkg/wekafs/mounter.go b/pkg/wekafs/wekafsmounter.go similarity index 53% rename from pkg/wekafs/mounter.go rename to pkg/wekafs/wekafsmounter.go index b736f1cab..615d9d8f6 100644 --- a/pkg/wekafs/mounter.go +++ b/pkg/wekafs/wekafsmounter.go @@ -1,13 +1,10 @@ package wekafs import ( - "bufio" "context" "github.com/rs/zerolog/log" "github.com/wekafs/csi-wekafs/pkg/wekafs/apiclient" "k8s.io/mount-utils" - "os" - "strings" "sync" "time" ) @@ -16,28 +13,34 @@ const ( inactiveMountGcPeriod = time.Minute * 10 ) -type mountsMapPerFs map[string]*wekaMount -type mountsMap map[string]mountsMapPerFs - -type wekaMounter struct { +type wekafsMounter struct { mountMap mountsMap lock sync.Mutex kMounter mount.Interface debugPath string - selinuxSupport bool + selinuxSupport *bool gc *innerPathVolGc allowProtocolContainers bool } -func newWekaMounter(driver *WekaFsDriver) *wekaMounter { - mounter := &wekaMounter{mountMap: mountsMap{}, debugPath: driver.debugPath, selinuxSupport: driver.selinuxSupport, allowProtocolContainers: driver.config.allowProtocolContainers} +func (m *wekafsMounter) getGarbageCollector() *innerPathVolGc { + return m.gc +} + +func newWekafsMounter(driver *WekaFsDriver) *wekafsMounter { + var selinuxSupport *bool + if driver.selinuxSupport { + log.Debug().Msg("SELinux support is forced") + selinuxSupport = &[]bool{true}[0] + } + mounter := &wekafsMounter{mountMap: mountsMap{}, debugPath: driver.debugPath, selinuxSupport: selinuxSupport} mounter.gc = initInnerPathVolumeGc(mounter) mounter.schedulePeriodicMountGc() return mounter } -func (m *wekaMounter) NewMount(fsName string, options MountOptions) *wekaMount { +func (m *wekafsMounter) NewMount(fsName string, options MountOptions) AnyMount { m.lock.Lock() if m.kMounter == nil { m.kMounter = mount.New("") @@ -45,9 +48,9 @@ func (m *wekaMounter) NewMount(fsName string, options MountOptions) *wekaMount { if _, ok := m.mountMap[fsName]; !ok { m.mountMap[fsName] = mountsMapPerFs{} } - if _, ok := m.mountMap[fsName][options.String()]; !ok { + if _, ok := m.mountMap[fsName][options.AsMapKey()]; !ok { uniqueId := getStringSha1AsB32(fsName + ":" + options.String()) - wMount := &wekaMount{ + wMount := &wekafsMount{ kMounter: m.kMounter, fsName: fsName, debugPath: m.debugPath, @@ -55,47 +58,23 @@ func (m *wekaMounter) NewMount(fsName string, options MountOptions) *wekaMount { mountOptions: options, allowProtocolContainers: m.allowProtocolContainers, } - m.mountMap[fsName][options.String()] = wMount + m.mountMap[fsName][options.AsMapKey()] = wMount } m.lock.Unlock() - return m.mountMap[fsName][options.String()] + return m.mountMap[fsName][options.AsMapKey()] } -type UnmountFunc func() - -func (m *wekaMounter) getSelinuxStatus(ctx context.Context) bool { - logger := log.Ctx(ctx) - if m.selinuxSupport { - logger.Trace().Msg("SELinux support is forced") +func (m *wekafsMounter) getSelinuxStatus(ctx context.Context) bool { + if m.selinuxSupport != nil && *m.selinuxSupport { return true } - // check if we have /etc/selinux/config - // if it exists, we can check if selinux is enforced or not - selinuxConf := "/etc/selinux/config" - file, err := os.Open(selinuxConf) - if err != nil { - return false - } - defer func() { _ = file.Close() }() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if strings.Contains(line, "SELINUX=enforcing") { - // no need to repeat each time, just set the selinuxSupport to true - m.selinuxSupport = true - return true - } - } - - if err := scanner.Err(); err != nil { - logger.Error().Err(err).Str("filename", selinuxConf).Msg("Failed to read SELinux config file") - } - return false + selinuxSupport := getSelinuxStatus(ctx) + m.selinuxSupport = &selinuxSupport + return *m.selinuxSupport } -func (m *wekaMounter) mountWithOptions(ctx context.Context, fsName string, mountOptions MountOptions, apiClient *apiclient.ApiClient) (string, error, UnmountFunc) { - mountOptions.setSelinux(m.getSelinuxStatus(ctx)) +func (m *wekafsMounter) mountWithOptions(ctx context.Context, fsName string, mountOptions MountOptions, apiClient *apiclient.ApiClient) (string, error, UnmountFunc) { + mountOptions.setSelinux(m.getSelinuxStatus(ctx), MountProtocolWekafs) mountObj := m.NewMount(fsName, mountOptions) mountErr := mountObj.incRef(ctx, apiClient) @@ -103,26 +82,26 @@ func (m *wekaMounter) mountWithOptions(ctx context.Context, fsName string, mount log.Ctx(ctx).Error().Err(mountErr).Msg("Failed mounting") return "", mountErr, func() {} } - return mountObj.mountPoint, nil, func() { + return mountObj.getMountPoint(), nil, func() { if mountErr == nil { _ = mountObj.decRef(ctx) } } } -func (m *wekaMounter) Mount(ctx context.Context, fs string, apiClient *apiclient.ApiClient) (string, error, UnmountFunc) { +func (m *wekafsMounter) Mount(ctx context.Context, fs string, apiClient *apiclient.ApiClient) (string, error, UnmountFunc) { return m.mountWithOptions(ctx, fs, getDefaultMountOptions(), apiClient) } -func (m *wekaMounter) unmountWithOptions(ctx context.Context, fsName string, options MountOptions) error { +func (m *wekafsMounter) unmountWithOptions(ctx context.Context, fsName string, options MountOptions) error { opts := options - options.setSelinux(m.getSelinuxStatus(ctx)) + options.setSelinux(m.getSelinuxStatus(ctx), MountProtocolWekafs) log.Ctx(ctx).Trace().Strs("mount_options", opts.Strings()).Str("filesystem", fsName).Msg("Received an unmount request") - if mnt, ok := m.mountMap[fsName][options.String()]; ok { + if mnt, ok := m.mountMap[fsName][options.AsMapKey()]; ok { err := mnt.decRef(ctx) if err == nil { - if m.mountMap[fsName][options.String()].refCount <= 0 { + if m.mountMap[fsName][options.AsMapKey()].getRefCount() <= 0 { log.Ctx(ctx).Trace().Str("filesystem", fsName).Strs("mount_options", options.Strings()).Msg("This is a last use of this mount, removing from map") delete(m.mountMap[fsName], options.String()) } @@ -139,17 +118,17 @@ func (m *wekaMounter) unmountWithOptions(ctx context.Context, fsName string, opt } } -func (m *wekaMounter) LogActiveMounts() { +func (m *wekafsMounter) LogActiveMounts() { if len(m.mountMap) > 0 { count := 0 for fsName := range m.mountMap { for mnt := range m.mountMap[fsName] { mapEntry := m.mountMap[fsName][mnt] - if mapEntry.refCount > 0 { - log.Trace().Str("filesystem", fsName).Int("refcount", mapEntry.refCount).Strs("mount_options", mapEntry.mountOptions.Strings()).Msg("Mount is active") + if mapEntry.getRefCount() > 0 { + log.Trace().Str("filesystem", fsName).Int("refcount", mapEntry.getRefCount()).Strs("mount_options", mapEntry.getMountOptions().Strings()).Msg("Mount is active") count++ } else { - log.Trace().Str("filesystem", fsName).Int("refcount", mapEntry.refCount).Strs("mount_options", mapEntry.mountOptions.Strings()).Msg("Mount is not active") + log.Trace().Str("filesystem", fsName).Int("refcount", mapEntry.getRefCount()).Strs("mount_options", mapEntry.getMountOptions().Strings()).Msg("Mount is not active") } } @@ -158,16 +137,16 @@ func (m *wekaMounter) LogActiveMounts() { } } -func (m *wekaMounter) gcInactiveMounts() { +func (m *wekafsMounter) gcInactiveMounts() { if len(m.mountMap) > 0 { for fsName := range m.mountMap { for uniqueId, wekaMount := range m.mountMap[fsName] { - if wekaMount.refCount == 0 { - if wekaMount.lastUsed.Before(time.Now().Add(-inactiveMountGcPeriod)) { + if wekaMount.getRefCount() == 0 { + if wekaMount.getLastUsed().Before(time.Now().Add(-inactiveMountGcPeriod)) { m.lock.Lock() - if wekaMount.refCount == 0 { - log.Trace().Str("filesystem", fsName).Strs("mount_options", wekaMount.mountOptions.Strings()). - Time("last_used", wekaMount.lastUsed).Msg("Removing stale mount from map") + if wekaMount.getRefCount() == 0 { + log.Trace().Str("filesystem", fsName).Strs("mount_options", wekaMount.getMountOptions().Strings()). + Time("last_used", wekaMount.getLastUsed()).Msg("Removing stale mount from map") delete(m.mountMap[fsName], uniqueId) } m.lock.Unlock() @@ -181,7 +160,7 @@ func (m *wekaMounter) gcInactiveMounts() { } } -func (m *wekaMounter) schedulePeriodicMountGc() { +func (m *wekafsMounter) schedulePeriodicMountGc() { go func() { log.Debug().Msg("Initializing periodic mount GC") for true {