diff --git a/eks-anywhere-vsphere/Addons/Core/storage-driver/openebs/helm-release.yaml b/eks-anywhere-vsphere/Addons/Core/storage-driver/openebs/helm-release.yaml deleted file mode 100644 index 58dee1e2..00000000 --- a/eks-anywhere-vsphere/Addons/Core/storage-driver/openebs/helm-release.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: openebs - namespace: openebs -spec: - chart: - spec: - chart: openebs - reconcileStrategy: ChartVersion - sourceRef: - kind: HelmRepository - name: openebs - namespace: openebs - version: "3.3.0" - interval: 60s - targetNamespace: openebs - diff --git a/eks-anywhere-vsphere/Addons/Core/storage-driver/openebs/helm-repo.yaml b/eks-anywhere-vsphere/Addons/Core/storage-driver/openebs/helm-repo.yaml deleted file mode 100644 index d9665841..00000000 --- a/eks-anywhere-vsphere/Addons/Core/storage-driver/openebs/helm-repo.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: source.toolkit.fluxcd.io/v1beta2 -kind: HelmRepository -metadata: - name: openebs - namespace: openebs -spec: - interval: 30s - url: https://openebs.github.io/charts - diff --git a/eks-anywhere-vsphere/Addons/Core/storage-driver/openebs/kustomization.yaml b/eks-anywhere-vsphere/Addons/Core/storage-driver/openebs/kustomization.yaml deleted file mode 100644 index de9f0a84..00000000 --- a/eks-anywhere-vsphere/Addons/Core/storage-driver/openebs/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - namespace.yaml - - helm-repo.yaml - - helm-release.yaml \ No newline at end of file diff --git a/eks-anywhere-vsphere/Addons/Core/storage-driver/storage-driver.yaml b/eks-anywhere-vsphere/Addons/Core/storage-driver/storage-driver.yaml index f6aa02b5..df0cd243 100644 --- a/eks-anywhere-vsphere/Addons/Core/storage-driver/storage-driver.yaml +++ b/eks-anywhere-vsphere/Addons/Core/storage-driver/storage-driver.yaml @@ -1,10 +1,10 @@ apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 kind: Kustomization metadata: - name: openebs + name: vmware-csi namespace: flux-system spec: - path: "./eks-anywhere-vsphere/Addons/Core/storage-driver/openebs" + path: "./eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi" sourceRef: kind: GitRepository name: addons diff --git a/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/README.md b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/README.md new file mode 100644 index 00000000..33388666 --- /dev/null +++ b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/README.md @@ -0,0 +1,46 @@ +## VMware CSI Driver Pre-Requisites + +In order to sucessfully install the VMware CSI Driver, the following items must be completed first: +1. Creation of the `ClusterSecretStore` Kubernetes resource detailed in the eks-anywhere-addons/README.md file + +2. Creation of config secrets in AWS Secrets Manager that External secrets will reference in the +vsphere-csi-externalsecrets.yaml file. + +# Creation of the Config Secrets + +Two Secrets must be created, csi-vsphere.conf referenced by the +vsphere-csi-controller in the vsphere-csi-driver.yaml file, and +vsphere.conf that is referenced in the vsphere-cloud-controller.yaml file + +Both secrets are multi-line configs, so the "Plaintext" secret type must be used. + +In csi-vsphere.conf, the secret will contain Global configuration data and Virtual Center specific information in the following format: + +``` +[Global] +insecure-flag = "true" +port = "443" + +[VirtualCenter ""] +cluster-id = ": + server: + user: + password: + datacenters: + - +``` + diff --git a/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/kustomization.yaml b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/kustomization.yaml new file mode 100644 index 00000000..c9bbb152 --- /dev/null +++ b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - vsphere-cloud-controller-manager.yaml + - vsphere-csi-externalsecrets.yaml + - vsphere-csi-driver.yaml + - vmware-sc.yaml \ No newline at end of file diff --git a/eks-anywhere-vsphere/Addons/Core/storage-driver/openebs/namespace.yaml b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/namespace.yaml similarity index 62% rename from eks-anywhere-vsphere/Addons/Core/storage-driver/openebs/namespace.yaml rename to eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/namespace.yaml index 21752858..f6b4f38b 100644 --- a/eks-anywhere-vsphere/Addons/Core/storage-driver/openebs/namespace.yaml +++ b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/namespace.yaml @@ -1,4 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - name: openebs \ No newline at end of file + name: vmware-system-csi \ No newline at end of file diff --git a/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/vmware-sc.yaml b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/vmware-sc.yaml new file mode 100644 index 00000000..14b2898c --- /dev/null +++ b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/vmware-sc.yaml @@ -0,0 +1,7 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: vmware-sc + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: csi.vsphere.vmware.com \ No newline at end of file diff --git a/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/vsphere-cloud-controller-manager.yaml b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/vsphere-cloud-controller-manager.yaml new file mode 100644 index 00000000..22880a6e --- /dev/null +++ b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/vsphere-cloud-controller-manager.yaml @@ -0,0 +1,207 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + labels: + vsphere-cpi-infra: service-account + component: cloud-controller-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: servicecatalog.k8s.io:apiserver-authentication-reader + labels: + vsphere-cpi-infra: role-binding + component: cloud-controller-manager + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:cloud-controller-manager + labels: + vsphere-cpi-infra: cluster-role-binding + component: cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:cloud-controller-manager + labels: + vsphere-cpi-infra: role + component: cloud-controller-manager +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - create + - get + - list + - watch + - update +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: vsphere-cloud-controller-manager + labels: + component: cloud-controller-manager + tier: control-plane + namespace: kube-system +spec: + selector: + matchLabels: + name: vsphere-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + name: vsphere-cloud-controller-manager + component: cloud-controller-manager + tier: control-plane + spec: + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + operator: Exists + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + operator: Exists + - key: node.kubernetes.io/not-ready + effect: NoSchedule + operator: Exists + securityContext: + runAsUser: 1001 + serviceAccountName: cloud-controller-manager + priorityClassName: system-node-critical + containers: + - name: vsphere-cloud-controller-manager + image: gcr.io/cloud-provider-vsphere/cpi/release/manager:v1.28.0 + args: + - --cloud-provider=vsphere + - --v=2 + - --cloud-config=/etc/cloud/vsphere.conf + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + resources: + requests: + cpu: 200m + hostNetwork: true + volumes: + - name: vsphere-config-volume + secret: + secretName: vsphere.conf + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists diff --git a/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/vsphere-csi-driver.yaml b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/vsphere-csi-driver.yaml new file mode 100644 index 00000000..44df9661 --- /dev/null +++ b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/vsphere-csi-driver.yaml @@ -0,0 +1,715 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.vsphere.vmware.com +spec: + attachRequired: true + podInfoOnMount: false +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: vsphere-csi-controller + namespace: vmware-system-csi +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vsphere-csi-controller-role +rules: + - apiGroups: [""] + resources: ["nodes", "pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["cns.vmware.com"] + resources: ["triggercsifullsyncs"] + verbs: ["create", "get", "update", "watch", "list"] + - apiGroups: ["cns.vmware.com"] + resources: ["cnsvspherevolumemigrations"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["cns.vmware.com"] + resources: ["cnsvolumeinfoes"] + verbs: ["create", "get", "list", "watch", "delete"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "create", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: ["cns.vmware.com"] + resources: ["cnsvolumeoperationrequests"] + verbs: ["create", "get", "list", "update", "delete"] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshots" ] + verbs: [ "get", "list" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotclasses" ] + verbs: [ "watch", "get", "list" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents" ] + verbs: [ "create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents/status" ] + verbs: [ "update", "patch" ] + - apiGroups: [ "cns.vmware.com" ] + resources: [ "csinodetopologies" ] + verbs: ["get", "update", "watch", "list"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vsphere-csi-controller-binding +subjects: + - kind: ServiceAccount + name: vsphere-csi-controller + namespace: vmware-system-csi +roleRef: + kind: ClusterRole + name: vsphere-csi-controller-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: vsphere-csi-node + namespace: vmware-system-csi +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vsphere-csi-node-cluster-role +rules: + - apiGroups: ["cns.vmware.com"] + resources: ["csinodetopologies"] + verbs: ["create", "watch", "get", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vsphere-csi-node-cluster-role-binding +subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi +roleRef: + kind: ClusterRole + name: vsphere-csi-node-cluster-role + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vsphere-csi-node-role + namespace: vmware-system-csi +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vsphere-csi-node-binding + namespace: vmware-system-csi +subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi +roleRef: + kind: Role + name: vsphere-csi-node-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +data: + "csi-migration": "true" + "csi-auth-check": "true" + "online-volume-extend": "true" + "trigger-csi-fullsync": "false" + "async-query-volume": "true" + "block-volume-snapshot": "true" + "csi-windows-support": "true" + "use-csinode-id": "true" + "list-volumes": "true" + "pv-to-backingdiskobjectid-mapping": "false" + "cnsmgr-suspend-create-volume": "true" + "topology-preferential-datastores": "true" + "max-pvscsi-targets-per-vm": "true" + "multi-vcenter-csi-topology": "true" + "csi-internal-generated-cluster-id": "true" + "listview-tasks": "false" +kind: ConfigMap +metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi +--- +apiVersion: v1 +kind: Service +metadata: + name: vsphere-csi-controller + namespace: vmware-system-csi + labels: + app: vsphere-csi-controller +spec: + ports: + - name: ctlr + port: 2112 + targetPort: 2112 + protocol: TCP + - name: syncer + port: 2113 + targetPort: 2113 + protocol: TCP + selector: + app: vsphere-csi-controller +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: vsphere-csi-controller + namespace: vmware-system-csi +spec: + replicas: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + selector: + matchLabels: + app: vsphere-csi-controller + template: + metadata: + labels: + app: vsphere-csi-controller + role: vsphere-csi + spec: + priorityClassName: system-cluster-critical # Guarantees scheduling for critical system pods + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - vsphere-csi-controller + topologyKey: "kubernetes.io/hostname" + serviceAccountName: vsphere-csi-controller + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + # uncomment below toleration if you need an aggressive pod eviction in case when + # node becomes not-ready or unreachable. Default is 300 seconds if not specified. + #- key: node.kubernetes.io/not-ready + # operator: Exists + # effect: NoExecute + # tolerationSeconds: 30 + #- key: node.kubernetes.io/unreachable + # operator: Exists + # effect: NoExecute + # tolerationSeconds: 30 + dnsPolicy: "Default" + containers: + - name: csi-attacher + image: k8s.gcr.io/sig-storage/csi-attacher:v4.2.0 + args: + - "--v=4" + - "--timeout=300s" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-lease-duration=120s" + - "--leader-election-renew-deadline=60s" + - "--leader-election-retry-period=30s" + - "--kube-api-qps=100" + - "--kube-api-burst=100" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + - name: csi-resizer + image: k8s.gcr.io/sig-storage/csi-resizer:v1.7.0 + args: + - "--v=4" + - "--timeout=300s" + - "--handle-volume-inuse-error=false" + - "--csi-address=$(ADDRESS)" + - "--kube-api-qps=100" + - "--kube-api-burst=100" + - "--leader-election" + - "--leader-election-lease-duration=120s" + - "--leader-election-renew-deadline=60s" + - "--leader-election-retry-period=30s" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + - name: vsphere-csi-controller + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.0.0 + args: + - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" + - "--fss-namespace=$(CSI_NAMESPACE)" + imagePullPolicy: "Always" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: X_CSI_MODE + value: "controller" + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m + - name: VSPHERE_CSI_CONFIG + value: "/etc/cloud/csi-vsphere.conf" + - name: LOGGER_LEVEL + value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + - mountPath: /csi + name: socket-dir + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + - name: prometheus + containerPort: 2112 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 180 + failureThreshold: 3 + - name: liveness-probe + image: k8s.gcr.io/sig-storage/livenessprobe:v2.9.0 + args: + - "--v=4" + - "--csi-address=/csi/csi.sock" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: vsphere-syncer + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.0.0 + args: + - "--leader-election" + - "--leader-election-lease-duration=120s" + - "--leader-election-renew-deadline=60s" + - "--leader-election-retry-period=30s" + - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" + - "--fss-namespace=$(CSI_NAMESPACE)" + imagePullPolicy: "Always" + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP + env: + - name: FULL_SYNC_INTERVAL_MINUTES + value: "30" + - name: VSPHERE_CSI_CONFIG + value: "/etc/cloud/csi-vsphere.conf" + - name: LOGGER_LEVEL + value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + - name: csi-provisioner + image: k8s.gcr.io/sig-storage/csi-provisioner:v3.4.0 + args: + - "--v=4" + - "--timeout=300s" + - "--csi-address=$(ADDRESS)" + - "--kube-api-qps=100" + - "--kube-api-burst=100" + - "--leader-election" + - "--leader-election-lease-duration=120s" + - "--leader-election-renew-deadline=60s" + - "--leader-election-retry-period=30s" + - "--default-fstype=ext4" + # needed only for topology aware setup + #- "--feature-gates=Topology=true" + #- "--strict-topology" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + - name: csi-snapshotter + image: k8s.gcr.io/sig-storage/csi-snapshotter:v6.2.1 + args: + - "--v=4" + - "--kube-api-qps=100" + - "--kube-api-burst=100" + - "--timeout=300s" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-lease-duration=120s" + - "--leader-election-renew-deadline=60s" + - "--leader-election-retry-period=30s" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - name: vsphere-config-volume + secret: + secretName: vsphere-config-secret + - name: socket-dir + emptyDir: {} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: vsphere-csi-node + namespace: vmware-system-csi +spec: + selector: + matchLabels: + app: vsphere-csi-node + updateStrategy: + type: "RollingUpdate" + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app: vsphere-csi-node + role: vsphere-csi + spec: + priorityClassName: system-node-critical + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: vsphere-csi-node + hostNetwork: true + dnsPolicy: "ClusterFirstWithHostNet" + containers: + - name: node-driver-registrar + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.7.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 + - name: vsphere-csi-node + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.0.0 + args: + - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" + - "--fss-namespace=$(CSI_NAMESPACE)" + imagePullPolicy: "Always" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" # Maximum number of volumes that controller can publish to the node. If value is not set or zero Kubernetes decide how many volumes can be published by the controller to the node. + - name: X_CSI_MODE + value: "node" + - name: X_CSI_SPEC_REQ_VALIDATION + value: "false" + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: LOGGER_LEVEL + value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: /var/lib/kubelet + # needed so that any mounts setup inside this container are + # propagated back to the host machine. + mountPropagation: "Bidirectional" + - name: device-dir + mountPath: /dev + - name: blocks-dir + mountPath: /sys/block + - name: sys-devices-dir + mountPath: /sys/devices + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 5 + periodSeconds: 5 + failureThreshold: 3 + - name: liveness-probe + image: k8s.gcr.io/sig-storage/livenessprobe:v2.9.0 + args: + - "--v=4" + - "--csi-address=/csi/csi.sock" + volumeMounts: + - name: plugin-dir + mountPath: /csi + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: device-dir + hostPath: + path: /dev + - name: blocks-dir + hostPath: + path: /sys/block + type: Directory + - name: sys-devices-dir + hostPath: + path: /sys/devices + type: Directory + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: vsphere-csi-node-windows + namespace: vmware-system-csi +spec: + selector: + matchLabels: + app: vsphere-csi-node-windows + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app: vsphere-csi-node-windows + role: vsphere-csi-windows + spec: + priorityClassName: system-node-critical + nodeSelector: + kubernetes.io/os: windows + serviceAccountName: vsphere-csi-node + containers: + - name: node-driver-registrar + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.7.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + env: + - name: ADDRESS + value: 'unix://C:\\csi\\csi.sock' + - name: DRIVER_REG_SOCK_PATH + value: 'C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock' + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + livenessProbe: + exec: + command: + - /csi-node-driver-registrar.exe + - --kubelet-registration-path=C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 + - name: vsphere-csi-node + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.0.0 + args: + - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" + - "--fss-namespace=$(CSI_NAMESPACE)" + imagePullPolicy: "Always" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: 'unix://C:\\csi\\csi.sock' + - name: MAX_VOLUMES_PER_NODE + value: "59" # Maximum number of volumes that controller can publish to the node. If value is not set or zero Kubernetes decide how many volumes can be published by the controller to the node. + - name: X_CSI_MODE + value: node + - name: X_CSI_SPEC_REQ_VALIDATION + value: 'false' + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: LOGGER_LEVEL + value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION + - name: X_CSI_LOG_LEVEL + value: DEBUG + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + volumeMounts: + - name: plugin-dir + mountPath: 'C:\csi' + - name: pods-mount-dir + mountPath: 'C:\var\lib\kubelet' + - name: csi-proxy-volume-v1 + mountPath: \\.\pipe\csi-proxy-volume-v1 + - name: csi-proxy-filesystem-v1 + mountPath: \\.\pipe\csi-proxy-filesystem-v1 + - name: csi-proxy-disk-v1 + mountPath: \\.\pipe\csi-proxy-disk-v1 + - name: csi-proxy-system-v1alpha1 + mountPath: \\.\pipe\csi-proxy-system-v1alpha1 + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 5 + periodSeconds: 5 + failureThreshold: 3 + - name: liveness-probe + image: k8s.gcr.io/sig-storage/livenessprobe:v2.9.0 + args: + - "--v=4" + - "--csi-address=/csi/csi.sock" + volumeMounts: + - name: plugin-dir + mountPath: /csi + volumes: + - name: registration-dir + hostPath: + path: 'C:\var\lib\kubelet\plugins_registry\' + type: Directory + - name: plugin-dir + hostPath: + path: 'C:\var\lib\kubelet\plugins\csi.vsphere.vmware.com\' + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: \var\lib\kubelet + type: Directory + - name: csi-proxy-disk-v1 + hostPath: + path: \\.\pipe\csi-proxy-disk-v1 + type: '' + - name: csi-proxy-volume-v1 + hostPath: + path: \\.\pipe\csi-proxy-volume-v1 + type: '' + - name: csi-proxy-filesystem-v1 + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1 + type: '' + - name: csi-proxy-system-v1alpha1 + hostPath: + path: \\.\pipe\csi-proxy-system-v1alpha1 + type: '' + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists \ No newline at end of file diff --git a/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/vsphere-csi-externalsecrets.yaml b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/vsphere-csi-externalsecrets.yaml new file mode 100644 index 00000000..0ba9931c --- /dev/null +++ b/eks-anywhere-vsphere/Addons/Core/storage-driver/vmware-csi/vsphere-csi-externalsecrets.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: vsphere-cloud-config + namespace: kube-system + labels: + vsphere-cpi-infra: config + component: cloud-controller-manager +spec: + refreshInterval: 1m + secretStoreRef: + name: eksa-secret-store #The secret store name we have just created. + kind: ClusterSecretStore + target: + name: vsphere.conf # Secret name in k8s + data: + - secretKey: vsphere.conf # which key it's going to be stored in + remoteRef: + key: vsphere.conf # External secret name goes here +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: vsphere-config-secret + namespace: vmware-system-csi +spec: + refreshInterval: 1m + secretStoreRef: + name: eksa-secret-store #The secret store name we have just created. + kind: ClusterSecretStore + target: + name: vsphere-config-secret # Secret name in k8s + data: + - secretKey: csi-vsphere.conf # which key it's going to be stored in + remoteRef: + key: csi-vsphere.conf # External secret name goes here \ No newline at end of file