diff --git a/Dockerfile b/Dockerfile index aa6464e644..2c79af3489 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.15.6 AS builder +FROM golang:1.16 AS builder WORKDIR /go/src/github.com/kubernetes-sigs/aws-ebs-csi-driver COPY . . RUN make diff --git a/Dockerfile.windows b/Dockerfile.windows new file mode 100644 index 0000000000..420096dd38 --- /dev/null +++ b/Dockerfile.windows @@ -0,0 +1,23 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM --platform=$BUILDPLATFORM golang:1.16 AS builder +WORKDIR /go/src/github.com/kubernetes-sigs/aws-ebs-csi-driver +COPY . . +RUN make bin/aws-ebs-csi-driver.exe + +FROM mcr.microsoft.com/windows/servercore:1809 +COPY --from=builder /go/src/github.com/kubernetes-sigs/aws-ebs-csi-driver/bin/aws-ebs-csi-driver.exe /aws-ebs-csi-driver.exe + +ENTRYPOINT ["/aws-ebs-csi-driver.exe"] diff --git a/Makefile b/Makefile index 91781e98c7..743eabc945 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,10 @@ GOBIN=$(shell pwd)/bin bin/aws-ebs-csi-driver: | bin CGO_ENABLED=0 GOOS=linux go build -mod=vendor -ldflags ${LDFLAGS} -o bin/aws-ebs-csi-driver ./cmd/ +.PHONY: bin/aws-ebs-csi-driver.exe +bin/aws-ebs-csi-driver.exe: | bin + CGO_ENABLED=0 GOOS=windows go build -mod=vendor -ldflags ${LDFLAGS} -o bin/aws-ebs-csi-driver.exe ./cmd/ + bin /tmp/helm /tmp/kubeval: @mkdir -p $@ diff --git a/charts/aws-ebs-csi-driver/templates/node-windows.yaml b/charts/aws-ebs-csi-driver/templates/node-windows.yaml new file mode 100644 index 0000000000..b6cde40d7b --- /dev/null +++ b/charts/aws-ebs-csi-driver/templates/node-windows.yaml @@ -0,0 +1,185 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ebs-csi-node-windows + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.node.podAnnotations }} + annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- end }} + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + nodeSelector: + kubernetes.io/os: windows + {{- with .Values.node.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + serviceAccountName: {{ .Values.serviceAccount.node.name }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-cluster-critical" }} + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- else }} + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 + {{- end }} + {{- with .Values.node.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: ebs-plugin + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + args: + - node + - --endpoint=$(CSI_ENDPOINT) + {{- if .Values.volumeAttachLimit }} + - --volume-attach-limit={{ .Values.volumeAttachLimit }} + {{- end }} + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: C:\var\lib\kubelet + mountPropagation: "None" + - name: plugin-dir + mountPath: C:\csi + - name: csi-proxy-disk-pipe + mountPath: \\.\pipe\csi-proxy-disk-v1beta2 + - name: csi-proxy-volume-pipe + mountPath: \\.\pipe\csi-proxy-volume-v1beta2 + - name: csi-proxy-filesystem-pipe + mountPath: \\.\pipe\csi-proxy-filesystem-v1beta1 + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: node-driver-registrar + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + env: + - name: ADDRESS + value: unix:/csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: C:\var\lib\kubelet\plugins\ebs.csi.aws.com\csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} + volumeMounts: + - name: plugin-dir + mountPath: C:\csi + - name: registration-dir + mountPath: C:\registration + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + args: + - --csi-address=unix:/csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: C:\csi + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: C:\var\lib\kubelet + type: Directory + - name: plugin-dir + hostPath: + path: C:\var\lib\kubelet\plugins\ebs.csi.aws.com + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: C:\var\lib\kubelet\plugins_registry + type: Directory + - name: csi-proxy-disk-pipe + hostPath: + path: \\.\pipe\csi-proxy-disk-v1beta2 + type: "" + - name: csi-proxy-volume-pipe + hostPath: + path: \\.\pipe\csi-proxy-volume-v1beta2 + type: "" + - name: csi-proxy-filesystem-pipe + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1beta1 + type: "" diff --git a/examples/kubernetes/windows/README.md b/examples/kubernetes/windows/README.md new file mode 100644 index 0000000000..06f0083ace --- /dev/null +++ b/examples/kubernetes/windows/README.md @@ -0,0 +1,74 @@ +## Windows + +**This example requires pre-release versions of csi-proxy and the driver that do not exist yet. It is intended for developers only for now. Only basic read/write (mount/unmount and attach/detach) functionality has been tested, other features like resize don't work yet.** + +This example shows how to create a EBS volume and consume it from a Windows container dynamically. + + +## Prerequisites + +1. A 1.18+ Windows node. Windows support has only been tested on 1.18 EKS Windows nodes. https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html +2. [csi-proxy](https://github.com/kubernetes-csi/csi-proxy) built from commit c201c0afb8f12ceac6d5d778270c2702ca563889 or newer installed on the Windows node. +3. The driver vX.Y.Z+ (TODO: no such version exists yet) Node plugin (DaemonSet) installed on the Windows node. +4. The driver vX.Y.Z+ (TODO: no such version exists yet) Controller plugin (Deployment) installed on a Linux node. The Controller hasn't been tested on Windows. + +## Usage + +1. Create a sample app along with the StorageClass and the PersistentVolumeClaim: +``` +kubectl apply -f specs/ +``` + +2. Validate the volume was created and `volumeHandle` contains an EBS volumeID: +``` +kubectl describe pv +``` + +3. Validate the pod can write data to the volume: +``` +kubectl exec -it windows-server-iis-7c5fc8f6c5-t5mk9 -- powershell + +PS C:\> New-Item -Path data -Name "testfile1.txt" -ItemType "file" -Value "This +is a text string." + + + Directory: C:\data + + +Mode LastWriteTime Length Name +---- ------------- ------ ---- +-a---- 4/7/2021 12:31 AM 22 testfile1.txt +``` + +4. Validate a different pod can read data from the volume: +``` +kubectl delete po windows-server-iis-7c5fc8f6c5-t5mk9 + +kubectl exec -it windows-server-iis-7c5fc8f6c5-j44qv -- powershell + +PS C:\> ls data + + + Directory: C:\data + + +Mode LastWriteTime Length Name +---- ------------- ------ ---- +-a---- 4/7/2021 12:31 AM 22 testfile1.txt +``` + +5. OPTIONAL: In case you want to run some e2e tests with Windows pods, make sure to cordon Linux nodes for the duration of the test and modify the vpc-admission-webhook so that the Pods created as part of the tests get scheduled to the Windows nodes. +``` +kubectl cordon -l kubernetes.io/os=linux +# edit the webhook such that OSLabelSelectorOverride=all, otherwise the webhook +# won't mutate Pods created by the test and they won't run +kubectl edit deployment -n kube-system vpc-admission-webhook +deployment.apps/vpc-admission-webhook edited +ginkgo -nodes=1 -v --focus="External.Storage.*default.fs.*should.store.data" ./tests/e2e-kubernetes/ -- -kubeconfig=$KUBECONFIG -gce-zone=us-west-2a -node-os-distro=windows +``` + +6. Cleanup resources: +``` +kubectl delete -f specs/ +kubectl uncordon -l kubernetes.io/os=linux +``` diff --git a/examples/kubernetes/windows/specs/windows.yaml b/examples/kubernetes/windows/specs/windows.yaml new file mode 100644 index 0000000000..566e1b47b8 --- /dev/null +++ b/examples/kubernetes/windows/specs/windows.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: windows-server-iis +spec: + selector: + matchLabels: + app: windows-server-iis + tier: backend + track: stable + replicas: 1 + template: + metadata: + labels: + app: windows-server-iis + tier: backend + track: stable + spec: + containers: + - name: windows-server-iis + image: mcr.microsoft.com/windows/servercore:1809 + ports: + - name: http + containerPort: 80 + imagePullPolicy: IfNotPresent + command: + - powershell.exe + - -command + - "Add-WindowsFeature Web-Server; Invoke-WebRequest -UseBasicParsing -Uri 'https://dotnetbinaries.blob.core.windows.net/servicemonitor/2.0.1.6/ServiceMonitor.exe' -OutFile 'C:\\ServiceMonitor.exe'; echo '

Hello EKS!!!

' > C:\\inetpub\\wwwroot\\default.html; C:\\ServiceMonitor.exe 'w3svc'; " + volumeMounts: + - name: persistent-storage + mountPath: C:\data + nodeSelector: + kubernetes.io/os: windows + volumes: + - name: persistent-storage + persistentVolumeClaim: + claimName: ebs-claim +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ebs-claim +spec: + storageClassName: windows + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 4Gi +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: windows +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer +parameters: + fstype: ntfs diff --git a/pkg/driver/mount.go b/pkg/driver/mount.go index f1ce7b48c8..0e3e938a0b 100644 --- a/pkg/driver/mount.go +++ b/pkg/driver/mount.go @@ -17,199 +17,40 @@ limitations under the License. package driver import ( - "fmt" - "k8s.io/klog" - "os" - "strconv" - "strings" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/mounter" mountutils "k8s.io/mount-utils" - utilexec "k8s.io/utils/exec" ) +type mountInterface = mountutils.Interface + +// Mounter is the interface implemented by NodeMounter. +// A mix & match of functions defined in upstream libraries. (FormatAndMount +// from struct SafeFormatAndMount, PathExists from an old edition of +// mount.Interface). Define it explicitly so that it can be mocked and to +// insulate from oft-changing upstream interfaces/structs type Mounter interface { - // Implemented by NodeMounter.SafeFormatAndMount - mountutils.Interface - FormatAndMount(source string, target string, fstype string, options []string) error + mountInterface - // Implemented by NodeMounter.SafeFormatAndMount.Exec - // TODO this won't make sense on Windows with csi-proxy - utilexec.Interface + FormatAndMount(source string, target string, fstype string, options []string) error - // Implemented by NodeMounter below GetDeviceNameFromMount(mountPath string) (string, int, error) - // TODO this won't make sense on Windows with csi-proxy MakeFile(path string) error MakeDir(path string) error PathExists(path string) (bool, error) NeedResize(devicePath string, deviceMountPath string) (bool, error) } +// NodeMounter implements Mounter. +// A superstruct of SafeFormatAndMount. type NodeMounter struct { - mountutils.SafeFormatAndMount - utilexec.Interface + *mountutils.SafeFormatAndMount } func newNodeMounter() (Mounter, error) { + // mounter.NewSafeMounter returns a SafeFormatAndMount safeMounter, err := mounter.NewSafeMounter() if err != nil { return nil, err } - return &NodeMounter{*safeMounter, safeMounter.Exec}, nil -} - -// GetDeviceNameFromMount returns the volume ID for a mount path. -func (m NodeMounter) GetDeviceNameFromMount(mountPath string) (string, int, error) { - return mountutils.GetDeviceNameFromMount(m, mountPath) -} - -// This function is mirrored in ./sanity_test.go to make sure sanity test covered this block of code -// Please mirror the change to func MakeFile in ./sanity_test.go -func (m *NodeMounter) MakeFile(path string) error { - f, err := os.OpenFile(path, os.O_CREATE, os.FileMode(0644)) - if err != nil { - if !os.IsExist(err) { - return err - } - } - if err = f.Close(); err != nil { - return err - } - return nil -} - -// This function is mirrored in ./sanity_test.go to make sure sanity test covered this block of code -// Please mirror the change to func MakeFile in ./sanity_test.go -func (m *NodeMounter) MakeDir(path string) error { - err := os.MkdirAll(path, os.FileMode(0755)) - if err != nil { - if !os.IsExist(err) { - return err - } - } - return nil -} - -// This function is mirrored in ./sanity_test.go to make sure sanity test covered this block of code -// Please mirror the change to func MakeFile in ./sanity_test.go -func (m *NodeMounter) PathExists(path string) (bool, error) { - return mountutils.PathExists(path) -} - -//TODO: use common util from vendor kubernetes/mount-util -func (m *NodeMounter) NeedResize(devicePath string, deviceMountPath string) (bool, error) { - // TODO(xiangLi) resize fs size on formatted file system following this PR https://github.com/kubernetes/kubernetes/pull/99223 - // Port the in-tree un-released change first, need to remove after in-tree release - deviceSize, err := m.getDeviceSize(devicePath) - if err != nil { - return false, err - } - var fsSize, blockSize uint64 - format, err := m.SafeFormatAndMount.GetDiskFormat(devicePath) - if err != nil { - formatErr := fmt.Errorf("ResizeFS.Resize - error checking format for device %s: %v", devicePath, err) - return false, formatErr - } - - // If disk has no format, there is no need to resize the disk because mkfs.* - // by default will use whole disk anyways. - if format == "" { - return false, nil - } - - klog.V(3).Infof("ResizeFs.needResize - checking mounted volume %s", devicePath) - switch format { - case "ext3", "ext4": - blockSize, fsSize, err = m.getExtSize(devicePath) - klog.V(5).Infof("Ext size: filesystem size=%d, block size=%d", fsSize, blockSize) - case "xfs": - blockSize, fsSize, err = m.getXFSSize(deviceMountPath) - klog.V(5).Infof("Xfs size: filesystem size=%d, block size=%d, err=%v", fsSize, blockSize, err) - default: - klog.Errorf("Not able to parse given filesystem info. fsType: %s, will not resize", format) - return false, fmt.Errorf("Could not parse fs info on given filesystem format: %s. Supported fs types are: xfs, ext3, ext4", format) - } - if err != nil { - return false, err - } - // Tolerate one block difference, just in case of rounding errors somewhere. - klog.V(5).Infof("Volume %s: device size=%d, filesystem size=%d, block size=%d", devicePath, deviceSize, fsSize, blockSize) - if deviceSize <= fsSize+blockSize { - return false, nil - } - return true, nil -} -func (m *NodeMounter) getDeviceSize(devicePath string) (uint64, error) { - output, err := m.SafeFormatAndMount.Exec.Command("blockdev", "--getsize64", devicePath).CombinedOutput() - outStr := strings.TrimSpace(string(output)) - if err != nil { - return 0, fmt.Errorf("failed to read size of device %s: %s: %s", devicePath, err, outStr) - } - size, err := strconv.ParseUint(outStr, 10, 64) - if err != nil { - return 0, fmt.Errorf("failed to parse size of device %s %s: %s", devicePath, outStr, err) - } - return size, nil -} - -func (m *NodeMounter) getExtSize(devicePath string) (uint64, uint64, error) { - output, err := m.SafeFormatAndMount.Exec.Command("dumpe2fs", "-h", devicePath).CombinedOutput() - if err != nil { - return 0, 0, fmt.Errorf("failed to read size of filesystem on %s: %s: %s", devicePath, err, string(output)) - } - - blockSize, blockCount, _ := m.parseFsInfoOutput(string(output), ":", "block size", "block count") - - if blockSize == 0 { - return 0, 0, fmt.Errorf("could not find block size of device %s", devicePath) - } - if blockCount == 0 { - return 0, 0, fmt.Errorf("could not find block count of device %s", devicePath) - } - return blockSize, blockSize * blockCount, nil -} - -func (m *NodeMounter) getXFSSize(devicePath string) (uint64, uint64, error) { - output, err := m.SafeFormatAndMount.Exec.Command("xfs_io", "-c", "statfs", devicePath).CombinedOutput() - if err != nil { - return 0, 0, fmt.Errorf("failed to read size of filesystem on %s: %s: %s", devicePath, err, string(output)) - } - - blockSize, blockCount, _ := m.parseFsInfoOutput(string(output), "=", "geom.bsize", "geom.datablocks") - - if blockSize == 0 { - return 0, 0, fmt.Errorf("could not find block size of device %s", devicePath) - } - if blockCount == 0 { - return 0, 0, fmt.Errorf("could not find block count of device %s", devicePath) - } - return blockSize, blockSize * blockCount, nil -} - -func (m *NodeMounter) parseFsInfoOutput(cmdOutput string, spliter string, blockSizeKey string, blockCountKey string) (uint64, uint64, error) { - lines := strings.Split(cmdOutput, "\n") - var blockSize, blockCount uint64 - var err error - - for _, line := range lines { - tokens := strings.Split(line, spliter) - if len(tokens) != 2 { - continue - } - key, value := strings.ToLower(strings.TrimSpace(tokens[0])), strings.ToLower(strings.TrimSpace(tokens[1])) - if key == blockSizeKey { - blockSize, err = strconv.ParseUint(value, 10, 64) - if err != nil { - return 0, 0, fmt.Errorf("failed to parse block size %s: %s", value, err) - } - } - if key == blockCountKey { - blockCount, err = strconv.ParseUint(value, 10, 64) - if err != nil { - return 0, 0, fmt.Errorf("failed to parse block count %s: %s", value, err) - } - } - } - return blockSize, blockCount, err + return &NodeMounter{safeMounter}, nil } diff --git a/pkg/driver/mount_linux.go b/pkg/driver/mount_linux.go new file mode 100644 index 0000000000..ea4427c549 --- /dev/null +++ b/pkg/driver/mount_linux.go @@ -0,0 +1,184 @@ +// +build linux + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "fmt" + "k8s.io/klog" + "os" + "strconv" + "strings" + + mountutils "k8s.io/mount-utils" +) + +// GetDeviceNameFromMount returns the volume ID for a mount path. +func (m NodeMounter) GetDeviceNameFromMount(mountPath string) (string, int, error) { + return mountutils.GetDeviceNameFromMount(m, mountPath) +} + +// This function is mirrored in ./sanity_test.go to make sure sanity test covered this block of code +// Please mirror the change to func MakeFile in ./sanity_test.go +func (m *NodeMounter) MakeFile(path string) error { + f, err := os.OpenFile(path, os.O_CREATE, os.FileMode(0644)) + if err != nil { + if !os.IsExist(err) { + return err + } + } + if err = f.Close(); err != nil { + return err + } + return nil +} + +// This function is mirrored in ./sanity_test.go to make sure sanity test covered this block of code +// Please mirror the change to func MakeFile in ./sanity_test.go +func (m *NodeMounter) MakeDir(path string) error { + err := os.MkdirAll(path, os.FileMode(0755)) + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +// This function is mirrored in ./sanity_test.go to make sure sanity test covered this block of code +// Please mirror the change to func MakeFile in ./sanity_test.go +func (m *NodeMounter) PathExists(path string) (bool, error) { + return mountutils.PathExists(path) +} + +//TODO: use common util from vendor kubernetes/mount-util +func (m *NodeMounter) NeedResize(devicePath string, deviceMountPath string) (bool, error) { + // TODO(xiangLi) resize fs size on formatted file system following this PR https://github.com/kubernetes/kubernetes/pull/99223 + // Port the in-tree un-released change first, need to remove after in-tree release + deviceSize, err := m.getDeviceSize(devicePath) + if err != nil { + return false, err + } + var fsSize, blockSize uint64 + format, err := m.SafeFormatAndMount.GetDiskFormat(devicePath) + if err != nil { + formatErr := fmt.Errorf("ResizeFS.Resize - error checking format for device %s: %v", devicePath, err) + return false, formatErr + } + + // If disk has no format, there is no need to resize the disk because mkfs.* + // by default will use whole disk anyways. + if format == "" { + return false, nil + } + + klog.V(3).Infof("ResizeFs.needResize - checking mounted volume %s", devicePath) + switch format { + case "ext3", "ext4": + blockSize, fsSize, err = m.getExtSize(devicePath) + klog.V(5).Infof("Ext size: filesystem size=%d, block size=%d", fsSize, blockSize) + case "xfs": + blockSize, fsSize, err = m.getXFSSize(deviceMountPath) + klog.V(5).Infof("Xfs size: filesystem size=%d, block size=%d, err=%v", fsSize, blockSize, err) + default: + klog.Errorf("Not able to parse given filesystem info. fsType: %s, will not resize", format) + return false, fmt.Errorf("Could not parse fs info on given filesystem format: %s. Supported fs types are: xfs, ext3, ext4", format) + } + if err != nil { + return false, err + } + // Tolerate one block difference, just in case of rounding errors somewhere. + klog.V(5).Infof("Volume %s: device size=%d, filesystem size=%d, block size=%d", devicePath, deviceSize, fsSize, blockSize) + if deviceSize <= fsSize+blockSize { + return false, nil + } + return true, nil +} +func (m *NodeMounter) getDeviceSize(devicePath string) (uint64, error) { + output, err := m.SafeFormatAndMount.Exec.Command("blockdev", "--getsize64", devicePath).CombinedOutput() + outStr := strings.TrimSpace(string(output)) + if err != nil { + return 0, fmt.Errorf("failed to read size of device %s: %s: %s", devicePath, err, outStr) + } + size, err := strconv.ParseUint(outStr, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse size of device %s %s: %s", devicePath, outStr, err) + } + return size, nil +} + +func (m *NodeMounter) getExtSize(devicePath string) (uint64, uint64, error) { + output, err := m.SafeFormatAndMount.Exec.Command("dumpe2fs", "-h", devicePath).CombinedOutput() + if err != nil { + return 0, 0, fmt.Errorf("failed to read size of filesystem on %s: %s: %s", devicePath, err, string(output)) + } + + blockSize, blockCount, _ := m.parseFsInfoOutput(string(output), ":", "block size", "block count") + + if blockSize == 0 { + return 0, 0, fmt.Errorf("could not find block size of device %s", devicePath) + } + if blockCount == 0 { + return 0, 0, fmt.Errorf("could not find block count of device %s", devicePath) + } + return blockSize, blockSize * blockCount, nil +} + +func (m *NodeMounter) getXFSSize(devicePath string) (uint64, uint64, error) { + output, err := m.SafeFormatAndMount.Exec.Command("xfs_io", "-c", "statfs", devicePath).CombinedOutput() + if err != nil { + return 0, 0, fmt.Errorf("failed to read size of filesystem on %s: %s: %s", devicePath, err, string(output)) + } + + blockSize, blockCount, _ := m.parseFsInfoOutput(string(output), "=", "geom.bsize", "geom.datablocks") + + if blockSize == 0 { + return 0, 0, fmt.Errorf("could not find block size of device %s", devicePath) + } + if blockCount == 0 { + return 0, 0, fmt.Errorf("could not find block count of device %s", devicePath) + } + return blockSize, blockSize * blockCount, nil +} + +func (m *NodeMounter) parseFsInfoOutput(cmdOutput string, spliter string, blockSizeKey string, blockCountKey string) (uint64, uint64, error) { + lines := strings.Split(cmdOutput, "\n") + var blockSize, blockCount uint64 + var err error + + for _, line := range lines { + tokens := strings.Split(line, spliter) + if len(tokens) != 2 { + continue + } + key, value := strings.ToLower(strings.TrimSpace(tokens[0])), strings.ToLower(strings.TrimSpace(tokens[1])) + if key == blockSizeKey { + blockSize, err = strconv.ParseUint(value, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("failed to parse block size %s: %s", value, err) + } + } + if key == blockCountKey { + blockCount, err = strconv.ParseUint(value, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("failed to parse block count %s: %s", value, err) + } + } + } + return blockSize, blockCount, err +} diff --git a/pkg/driver/mount_test.go b/pkg/driver/mount_test.go index 18354e9442..32eb5a95f0 100644 --- a/pkg/driver/mount_test.go +++ b/pkg/driver/mount_test.go @@ -234,7 +234,7 @@ Journal checksum: 0xb7df3c6e Interface: mount.New(""), Exec: &fexec, } - fakeMounter := NodeMounter{safe, &fexec} + fakeMounter := NodeMounter{&safe} var blockSize uint64 var fsSize uint64 @@ -296,9 +296,9 @@ func TestNeedResize(t *testing.T) { } safe := mount.SafeFormatAndMount{ Interface: mount.New(""), - Exec: utilexec.New(), + Exec: &fexec, } - fakeMounter := NodeMounter{safe, &fexec} + fakeMounter := NodeMounter{&safe} needResize, err := fakeMounter.NeedResize(test.devicePath, test.deviceMountPath) if needResize != test.expectResult { diff --git a/pkg/driver/mount_windows.go b/pkg/driver/mount_windows.go new file mode 100644 index 0000000000..916672a136 --- /dev/null +++ b/pkg/driver/mount_windows.go @@ -0,0 +1,99 @@ +// +build windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "fmt" + "regexp" + + "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/mounter" +) + +func (m NodeMounter) FormatAndMount(source string, target string, fstype string, options []string) error { + proxyMounter, ok := m.SafeFormatAndMount.Interface.(*mounter.CSIProxyMounter) + if !ok { + return fmt.Errorf("failed to cast mounter to csi proxy mounter") + } + return proxyMounter.FormatAndMount(source, target, fstype, options) +} + +// GetDeviceNameFromMount returns the volume ID for a mount path. +// The ref count returned is always 1 or 0 because csi-proxy doesn't provide a +// way to determine the actual ref count (as opposed to Linux where the mount +// table gets read). In practice this shouldn't matter, as in the NodeStage +// case the ref count is ignored and in the NodeUnstage case, the ref count +// being >1 is just a warning. +// Command to determine ref count would be something like: +// Get-Volume -UniqueId "\\?\Volume{7c3da0c1-0000-0000-0000-010000000000}\" | Get-Partition | Select AccessPaths +func (m NodeMounter) GetDeviceNameFromMount(mountPath string) (string, int, error) { + proxyMounter, ok := m.SafeFormatAndMount.Interface.(*mounter.CSIProxyMounter) + if !ok { + return "", 0, fmt.Errorf("failed to cast mounter to csi proxy mounter") + } + deviceName, err := proxyMounter.GetDeviceNameFromMount(mountPath, "") + if err != nil { + // HACK change csi-proxy behavior instead of relying on fragile internal + // implementation details! + // if err contains '"(Get-Item...).Target, output: , error: ' then the + // internal Get-Item cmdlet didn't fail but no item/device was found at the + // path so we should return empty string and nil error just like the Linux + // implementation would. + pattern := `(Get-Item -Path \S+).Target, output: , error: ` + matched, matchErr := regexp.MatchString(pattern, err.Error()) + if matched { + return "", 0, nil + } + err = fmt.Errorf("error getting device name from mount: %v", err) + if matchErr != nil { + err = fmt.Errorf("%v, and error matching pattern %q: %v", err, pattern, matchErr) + } + return "", 0, err + } + return deviceName, 1, nil +} + +func (m *NodeMounter) MakeFile(path string) error { + proxyMounter, ok := m.SafeFormatAndMount.Interface.(*mounter.CSIProxyMounter) + if !ok { + return fmt.Errorf("failed to cast mounter to csi proxy mounter") + } + return proxyMounter.MakeFile(path) +} + +func (m *NodeMounter) MakeDir(path string) error { + proxyMounter, ok := m.SafeFormatAndMount.Interface.(*mounter.CSIProxyMounter) + if !ok { + return fmt.Errorf("failed to cast mounter to csi proxy mounter") + } + return proxyMounter.MakeDir(path) +} + +func (m *NodeMounter) PathExists(path string) (bool, error) { + proxyMounter, ok := m.SafeFormatAndMount.Interface.(*mounter.CSIProxyMounter) + if !ok { + return false, fmt.Errorf("failed to cast mounter to csi proxy mounter") + } + return proxyMounter.ExistsPath(path) +} + +func (m *NodeMounter) NeedResize(devicePath string, deviceMountPath string) (bool, error) { + // TODO this is called at NodeStage to ensure file system is the correct size + // Implement it to respect spec v1.4.0 https://github.com/container-storage-interface/spec/pull/452 + return false, nil +} diff --git a/pkg/driver/node.go b/pkg/driver/node.go index c844cc4416..641e16a732 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -218,7 +218,7 @@ func (d *nodeService) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, status.Errorf(codes.Internal, "Could not determine if volume %q (%q) need to be resized: %v", req.GetVolumeId(), source, err) } if needResize { - r := mountutils.NewResizeFs(d.mounter) + r := mountutils.NewResizeFs(d.mounter.(*NodeMounter).Exec) klog.V(2).Infof("Volume %s needs resizing", source) if _, err := r.Resize(source, target); err != nil { return nil, status.Errorf(codes.Internal, "Could not resize volume %q (%q): %v", volumeID, source, err) @@ -321,7 +321,7 @@ func (d *nodeService) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV // TODO this won't make sense on Windows with csi-proxy args := []string{"-o", "source", "--noheadings", "--target", volumePath} - output, err := d.mounter.Command("findmnt", args...).Output() + output, err := d.mounter.(*NodeMounter).Exec.Command("findmnt", args...).Output() if err != nil { return nil, status.Errorf(codes.Internal, "Could not determine device path: %v", err) @@ -331,7 +331,7 @@ func (d *nodeService) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV return nil, status.Errorf(codes.Internal, "Could not get valid device for mount path: %q", req.GetVolumePath()) } - r := mountutils.NewResizeFs(d.mounter) + r := mountutils.NewResizeFs(d.mounter.(*NodeMounter).Exec) // TODO: lock per volume ID to have some idempotency if _, err := r.Resize(devicePath, volumePath); err != nil { @@ -608,9 +608,8 @@ func (d *nodeService) nodePublishVolumeForFileSystem(req *csi.NodePublishVolumeR } } - klog.V(4).Infof("NodePublishVolume: creating dir %s", target) - if err := d.mounter.MakeDir(target); err != nil { - return status.Errorf(codes.Internal, "Could not create dir %q: %v", target, err) + if err := d.preparePublishTarget(target); err != nil { + return status.Errorf(codes.Internal, err.Error()) } fsType := mode.Mount.GetFsType() diff --git a/pkg/driver/node_linux.go b/pkg/driver/node_linux.go index 4c4a9d0955..a6efa9b305 100644 --- a/pkg/driver/node_linux.go +++ b/pkg/driver/node_linux.go @@ -26,7 +26,7 @@ import ( "strings" "golang.org/x/sys/unix" - "k8s.io/klog" + "k8s.io/klog/v2" ) // findDevicePath finds path of device and verifies its existence @@ -93,6 +93,14 @@ func findNvmeVolume(findName string) (device string, err error) { return resolved, nil } +func (d *nodeService) preparePublishTarget(target string) error { + klog.V(4).Infof("NodePublishVolume: creating dir %s", target) + if err := d.mounter.MakeDir(target); err != nil { + return fmt.Errorf("Could not create dir %q: %v", target, err) + } + return nil +} + // IsBlock checks if the given path is a block device func (d *nodeService) IsBlockDevice(fullPath string) (bool, error) { var st unix.Stat_t @@ -105,7 +113,7 @@ func (d *nodeService) IsBlockDevice(fullPath string) (bool, error) { } func (d *nodeService) getBlockSizeBytes(devicePath string) (int64, error) { - cmd := d.mounter.Command("blockdev", "--getsize64", devicePath) + cmd := d.mounter.(*NodeMounter).Exec.Command("blockdev", "--getsize64", devicePath) output, err := cmd.Output() if err != nil { return -1, fmt.Errorf("error when getting size of block volume at path %s: output: %s, err: %v", devicePath, string(output), err) diff --git a/pkg/driver/node_windows.go b/pkg/driver/node_windows.go new file mode 100644 index 0000000000..715a403a72 --- /dev/null +++ b/pkg/driver/node_windows.go @@ -0,0 +1,98 @@ +// +build windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "context" + "errors" + "fmt" + "strings" + + diskapi "github.com/kubernetes-csi/csi-proxy/client/api/disk/v1beta2" + diskclient "github.com/kubernetes-csi/csi-proxy/client/groups/disk/v1beta2" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/mounter" + "k8s.io/klog" +) + +// findDevicePath finds disk number of device +// https://docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/ec2-windows-volumes.html#list-nvme-powershell +func (d *nodeService) findDevicePath(devicePath, volumeID, _ string) (string, error) { + + diskClient, err := diskclient.NewClient() + if err != nil { + return "", fmt.Errorf("error creating csi-proxy disk client: %q", err) + } + defer diskClient.Close() + + response, err := diskClient.ListDiskIDs(context.TODO(), &diskapi.ListDiskIDsRequest{}) + if err != nil { + return "", fmt.Errorf("error listing disk ids: %q", err) + } + + diskIDs := response.GetDiskIDs() + + foundDiskNumber := "" + for diskNumber, diskID := range diskIDs { + serialNumber := diskID.Identifiers["serialNumber"] + cleanVolumeID := strings.ReplaceAll(volumeID, "-", "") + if strings.Contains(serialNumber, cleanVolumeID) { + foundDiskNumber = diskNumber + break + } + } + + if foundDiskNumber == "" { + return "", fmt.Errorf("disk number for device path %q volume id %q not found", devicePath, volumeID) + } + + return foundDiskNumber, nil +} + +func (d *nodeService) preparePublishTarget(target string) error { + // On Windows, Mount will create the parent of target and mklink (create a symbolic link) at target later, so don't create a + // directory at target now. Otherwise mklink will error: "Cannot create a file when that file already exists". + // Instead, delete the target if it already exists (like if it was created by kubelet <1.20) + // https://github.com/kubernetes/kubernetes/pull/88759 + klog.V(4).Infof("NodePublishVolume: removing dir %s", target) + exists, err := d.mounter.PathExists(target) + if err != nil { + return fmt.Errorf("error checking path %q exists: %v", target, err) + } + + proxyMounter, ok := (d.mounter.(*NodeMounter)).SafeFormatAndMount.Interface.(*mounter.CSIProxyMounter) + if !ok { + return fmt.Errorf("failed to cast mounter to csi proxy mounter") + } + + if exists { + if err := proxyMounter.Rmdir(target); err != nil { + return fmt.Errorf("error Rmdir target %q: %v", target, err) + } + } + return nil +} + +// IsBlock checks if the given path is a block device +func (d *nodeService) IsBlockDevice(fullPath string) (bool, error) { + return false, errors.New("unsupported") +} + +func (d *nodeService) getBlockSizeBytes(devicePath string) (int64, error) { + return 0, errors.New("unsupported") +}