Fix etcd volume for non-root etcd containers

Kubernetes volumes are mounted for uid 0 by default. This meant that non-root containers could not write to these volumes. Our etcd deployment needs to persist data, so running etcd as non-root user would not work. This commit introduces two separate fixes for this issues: 1. When using the default security context, fsGroup is set to 1000. Newly provisioned volumes will be "chgrp -R"-ed to that gid. This fixes above issue in case the storage provider supports it. 2. When using our pv-hostpath chart, 1) would not work, as hostpath volumes _always_ keep their host permissions. To make this work, a new helm hook is added that runs a job on volume creation, manually setting the host volumes permission.
piraeusdatastore · Sep 22, 2020 · cc3af23 · cc3af23
1 parent 93cb75c
commit cc3af23
Show file tree

Hide file tree

Showing 13 changed files with 76 additions and 34 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+* The `pv-hostpath` helper chart automatically sets up permissions for non-root etcd containers.
 * Disable securityContext enforcement by setting `global.setSecurityContext=false`.
 * Add cluster roles to work with OpenShift's SCC system.
 * Control volume placement and accessibility by using CSIs Topology feature. Controlled by setting

diff --git a/charts/piraeus/charts/csi-snapshotter/templates/deployment.yaml b/charts/piraeus/charts/csi-snapshotter/templates/deployment.yaml
@@ -22,10 +22,10 @@ spec:
         app: snapshot-controller
     spec:
       serviceAccountName: snapshot-controller
+      securityContext:
+{{ include "operator.podsecuritycontext" . | indent 8 }}
       containers:
         - name: snapshot-controller
-          securityContext:
-{{ include "operator.securitycontext" . | indent 12 }}
           image: {{ .Values.image }}
           args:
             - "--v=5"

diff --git a/charts/piraeus/charts/etcd/templates/statefulset.yaml b/charts/piraeus/charts/etcd/templates/statefulset.yaml
@@ -41,12 +41,12 @@ spec:
       tolerations:
 {{ toYaml .Values.tolerations | indent 8 }}
 {{- end }}
+      securityContext:
+{{ include "operator.podsecuritycontext" . | indent 8 }}
       containers:
       - name: {{ template "etcd.fullname" . }}
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         imagePullPolicy: {{ .Values.global.imagePullPolicy | quote }}
-        securityContext:
-{{ include "operator.securitycontext" . | indent 10 }}
         ports:
         - containerPort: {{ .Values.peerPort }}
           name: peer

diff --git a/charts/piraeus/templates/_helpers.tpl b/charts/piraeus/templates/_helpers.tpl
@@ -82,10 +82,10 @@ Environment to pass to stork
 {{/*
 Sets the default security context for all pods
 */}}
-{{- define "operator.securitycontext" -}}
+{{- define "operator.podsecuritycontext" -}}
 {{- if .Values.global.setSecurityContext -}}
 runAsUser: 1000
 runAsGroup: 1000
-privileged: false
+fsGroup: 1000
 {{- end -}}
 {{- end -}}
diff --git a/charts/piraeus/templates/operator-deployment.yaml b/charts/piraeus/templates/operator-deployment.yaml
@@ -15,12 +15,12 @@ spec:
         name: {{ template "operator.fullname" . }}-operator
     spec:
       serviceAccountName: {{ template "operator.fullname" . }}
+      securityContext:
+{{ include "operator.podsecuritycontext" . | indent 8 }}
       containers:
         - name: piraeus-operator
           image: {{ .Values.operator.image }}
           imagePullPolicy: {{ .Values.global.imagePullPolicy | quote }}
-          securityContext:
-{{ include "operator.securitycontext" . | indent 12 }}
           env:
             - name: WATCH_NAMESPACE
               valueFrom:

diff --git a/charts/piraeus/templates/stork-deployment.yaml b/charts/piraeus/templates/stork-deployment.yaml
@@ -122,8 +122,6 @@ spec:
               protocol: TCP
           resources:
 {{ .Values.stork.storkResources | toYaml | indent 12}}
-          securityContext:
-{{ include "operator.securitycontext" . | indent 12 }}
           name: stork
           env:
 {{ include "stork.config" . | indent 12 }}
@@ -138,6 +136,8 @@ spec:
                       - stork
               topologyKey: "kubernetes.io/hostname"
       serviceAccountName: stork-account
+      securityContext:
+{{ include "operator.podsecuritycontext" . | indent 8 }}
 # Source: https://raw.githubusercontent.com/libopenstorage/stork/v2.4.1/specs/stork-scheduler.yaml
 ---
 apiVersion: v1
@@ -256,8 +256,6 @@ spec:
               port: 10251
             initialDelaySeconds: 15
           name: {{ .Release.Name }}-stork-scheduler
-          securityContext:
-{{ include "operator.securitycontext" . | indent 12 }}
           readinessProbe:
             httpGet:
               path: /healthz
@@ -275,5 +273,7 @@ spec:
                       - {{ .Release.Name }}-stork-scheduler
               topologyKey: "kubernetes.io/hostname"
       hostPID: false
+      securityContext:
+{{ include "operator.podsecuritycontext" . | indent 8 }}
       serviceAccountName: stork-scheduler-account
 {{- end }}
diff --git a/charts/pv-hostpath/Chart.yaml b/charts/pv-hostpath/Chart.yaml
@@ -1,3 +1,4 @@
 name: pv-hostpath
 description: Hostpath volumes for etcd persistence
-version: 0.2.0
+version: 0.2.1
+appVersion: 0.2.1
diff --git a/charts/pv-hostpath/templates/pv.yaml b/charts/pv-hostpath/templates/pv.yaml
@@ -11,7 +11,8 @@ spec:
   accessModes:
   - ReadWriteOnce
   hostPath:
-    path: {{ required "A valid .Values.size entry required!" $.Values.path }}
+    path: {{ required "A valid .Values.path entry required!" $.Values.path }}
+    type: DirectoryOrCreate
   nodeAffinity:
     required:
       nodeSelectorTerms:
@@ -21,4 +22,41 @@ spec:
           values:
           - {{ . }}
 ---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: {{ $.Release.Service | quote }}
+    app.kubernetes.io/instance: {{ $.Release.Name | quote }}
+    app.kubernetes.io/version: {{ $.Chart.AppVersion }}
+    helm.sh/chart: "{{ $.Chart.Name }}-{{ $.Chart.Version }}"
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-delete-policy": hook-succeeded
+  name: {{ $.Release.Name }}-{{ . }}-chown
+  namespace: {{ $.Release.Namespace | default "default" }}
+spec:
+  template:
+    metadata:
+      name: {{ $.Release.Name }}-{{ . }}-chown
+      namespace: {{ $.Release.Namespace | default "default" }}
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: chown
+          image: busybox
+          command:
+            - chown
+            - -R
+            - {{ $.Values.uid | default 1000 }}:{{ $.Values.uid | default 1000 }}
+            - /volume
+          volumeMounts:
+            - mountPath: /volume
+              name: pv
+      volumes:
+        - name: pv
+          hostPath:
+            type: DirectoryOrCreate
+            path: {{ required "A valid .Values.path entry required!" $.Values.path }}
+---
 {{ end }}
diff --git a/charts/pv-hostpath/values.yaml b/charts/pv-hostpath/values.yaml
@@ -1,2 +1,4 @@
 size: 1Gi
 path: /var/lib/linstor-etcd
+uid: 1000
+gid: 1000
diff --git a/deploy/piraeus/charts/csi-snapshotter/templates/deployment.yaml b/deploy/piraeus/charts/csi-snapshotter/templates/deployment.yaml
@@ -16,12 +16,12 @@ spec:
         app: snapshot-controller
     spec:
       serviceAccountName: snapshot-controller
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
       containers:
         - name: snapshot-controller
-          securityContext:
-            runAsUser: 1000
-            runAsGroup: 1000
-            privileged: false
           image: quay.io/k8scsi/snapshot-controller:v2.1.0
           args:
             - "--v=5"

diff --git a/deploy/piraeus/charts/etcd/templates/statefulset.yaml b/deploy/piraeus/charts/etcd/templates/statefulset.yaml
@@ -38,14 +38,14 @@ spec:
                 values:
                 - etcd
             topologyKey: kubernetes.io/hostname
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
       containers:
       - name: piraeus-op-etcd
         image: "gcr.io/etcd-development/etcd:v3.4.9"
         imagePullPolicy: "IfNotPresent"
-        securityContext:
-          runAsUser: 1000
-          runAsGroup: 1000
-          privileged: false
         ports:
         - containerPort: 2380
           name: peer

diff --git a/deploy/piraeus/templates/operator-deployment.yaml b/deploy/piraeus/templates/operator-deployment.yaml
@@ -17,14 +17,14 @@ spec:
         name: piraeus-op-operator
     spec:
       serviceAccountName: piraeus-op
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
       containers:
         - name: piraeus-operator
           image: quay.io/piraeusdatastore/piraeus-operator:latest
           imagePullPolicy: "IfNotPresent"
-          securityContext:
-            runAsUser: 1000
-            runAsGroup: 1000
-            privileged: false
           env:
             - name: WATCH_NAMESPACE
               valueFrom:

diff --git a/deploy/piraeus/templates/stork-deployment.yaml b/deploy/piraeus/templates/stork-deployment.yaml
@@ -206,10 +206,6 @@ spec:
               protocol: TCP
           resources:
             {}
-          securityContext:
-            runAsUser: 1000
-            runAsGroup: 1000
-            privileged: false
           name: stork
           env:
             - name: LS_CONTROLLERS
@@ -225,6 +221,10 @@ spec:
                       - stork
               topologyKey: "kubernetes.io/hostname"
       serviceAccountName: stork-account
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
 # Source: https://raw.githubusercontent.com/libopenstorage/stork/v2.4.1/specs/stork-scheduler.yaml
 ---
 # Source: piraeus/templates/stork-deployment.yaml
@@ -267,10 +267,6 @@ spec:
               port: 10251
             initialDelaySeconds: 15
           name: piraeus-op-stork-scheduler
-          securityContext:
-            runAsUser: 1000
-            runAsGroup: 1000
-            privileged: false
           readinessProbe:
             httpGet:
               path: /healthz
@@ -288,6 +284,10 @@ spec:
                       - piraeus-op-stork-scheduler
               topologyKey: "kubernetes.io/hostname"
       hostPID: false
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
       serviceAccountName: stork-scheduler-account
 ---
 # Source: piraeus/templates/stork-deployment.yaml