Add minimal support for PSPs

PodSecurityPolicies are used to control access to security related settings in deployments. We require 2 security contexts for our containers: * unprivileged: Can run as non-root user without sys privileges. Containers in this category are: * operator * linstor controller * stork + scheduler * etcd * csi controller * csi snapshot controller * privileged: Run as root with sys privileges. Containers in this category are: * linstor satellite + kmod injector * csi node To simplify handling of PSPs, this commit adds the following changes * all pods are associated with a service account * helm can generate the rolebindings that associate a serviceaccount with a PSP role.
piraeusdatastore · Aug 31, 2020 · ff85ce9 · ff85ce9
1 parent 3fb5c8e
commit ff85ce9
Show file tree

Hide file tree

Showing 25 changed files with 175 additions and 25 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+* All pods use a dedicated service account to allow for fine-grained permission control.
+* The new [helm section `psp.*`](./doc/helm-values.adoc#psp) can automatically configure the ServiceAccount
+  of all components to use the appropriate [PSP](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) roles.
+
+### Changed
+
+* linstor-controller no longer starts in a privileged container.
+
 ## [v1.0.0] - 2020-08-06
 
 ### Added

diff --git a/Makefile b/Makefile
@@ -94,4 +94,4 @@ release:
 deploy/piraeus:
 	rm -rf "$@"
 	mkdir -p "$@"
-	helm template piraeus-op charts/piraeus --output-dir deploy >/dev/null
+	helm template piraeus-op charts/piraeus --set stork.schedulerTag=v1.16.0 --output-dir deploy >/dev/null
diff --git a/charts/piraeus/charts/csi-snapshotter/templates/deployment.yaml b/charts/piraeus/charts/csi-snapshotter/templates/deployment.yaml
@@ -24,6 +24,10 @@ spec:
       serviceAccountName: snapshot-controller
       containers:
         - name: snapshot-controller
+          securityContext:
+            runAsUser: 1000
+            runAsGroup: 1000
+            privileged: false
           image: {{ .Values.image }}
           args:
             - "--v=5"

diff --git a/charts/piraeus/charts/etcd/templates/rbac.yaml b/charts/piraeus/charts/etcd/templates/rbac.yaml
@@ -0,0 +1,8 @@
+# This YAML file contains all RBAC objects that are necessary to run a
+# etcd
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: etcd
+  namespace: {{ .Release.Namespace }}
diff --git a/charts/piraeus/charts/etcd/templates/statefulset.yaml b/charts/piraeus/charts/etcd/templates/statefulset.yaml
@@ -28,6 +28,7 @@ spec:
         chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
         app: {{ template "etcd.name" . }}
     spec:
+      serviceAccountName: "etcd"
 {{- if .Values.affinity }}
       affinity:
 {{ toYaml .Values.affinity | indent 8 }}
@@ -44,6 +45,10 @@ spec:
       - name: {{ template "etcd.fullname" . }}
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         imagePullPolicy: {{ .Values.global.imagePullPolicy | quote }}
+        securityContext:
+          runAsUser: 1000
+          runAsGroup: 1000
+          privileged: false
         ports:
         - containerPort: {{ .Values.peerPort }}
           name: peer

diff --git a/charts/piraeus/templates/controller-rbac.yml b/charts/piraeus/templates/controller-rbac.yml
@@ -0,0 +1,8 @@
+# This YAML file contains all RBAC objects that are necessary to run a
+# LINSTOR controller pod
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: linstor-controller
+  namespace: {{ .Release.Namespace }}
diff --git a/charts/piraeus/templates/etcd-configmap.yaml b/charts/piraeus/templates/etcd-configmap.yaml
diff --git a/charts/piraeus/templates/operator-deployment.yaml b/charts/piraeus/templates/operator-deployment.yaml
@@ -19,6 +19,10 @@ spec:
         - name: piraeus-operator
           image: {{ .Values.operator.image }}
           imagePullPolicy: {{ .Values.global.imagePullPolicy | quote }}
+          securityContext:
+            runAsUser: 1000
+            runAsGroup: 1000
+            privileged: false
           env:
             - name: WATCH_NAMESPACE
               valueFrom:

diff --git a/charts/piraeus/templates/psp.yaml b/charts/piraeus/templates/psp.yaml
@@ -0,0 +1,31 @@
+{{- define "bind-role" -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Prefix }}-{{ .Role }}
+  namespace: {{ .Namespace }}
+roleRef:
+  kind: Role
+  name: {{ .Role }}
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  # Authorize specific service accounts:
+  {{- range .Accounts }}
+  - kind: ServiceAccount
+    name: {{ . }}
+    namespace: {{ $.Namespace }}
+  {{- end }}
+{{- end -}}
+
+{{ if .Values.psp.unprivilegedRole }}
+---
+{{ $unprivAccounts := list (include "operator.fullname" .) "linstor-controller" "csi-controller" "stork-scheduler-account" "stork-account" "snapshot-controller" "etcd" }}
+{{ template "bind-role" (dict "Prefix" .Release.Name "Role" .Values.psp.unprivilegedRole "Namespace" .Release.Namespace "Accounts" $unprivAccounts) }}
+---
+{{ end }}
+{{ if .Values.psp.privilegedRole }}
+---
+{{ $privAccounts := list "linstor-satellite" "csi-node" }}
+{{ template "bind-role" (dict "Prefix" .Release.Name "Role" .Values.psp.privilegedRole "Namespace" .Release.Namespace "Accounts" $privAccounts) }}
+---
+{{ end }}
diff --git a/charts/piraeus/templates/satellite-rbac.yml b/charts/piraeus/templates/satellite-rbac.yml
@@ -0,0 +1,8 @@
+# This YAML file contains all RBAC objects that are necessary to run a
+# LINSTOR satellite pod
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: linstor-satellite
+  namespace: {{ .Release.Namespace }}
diff --git a/charts/piraeus/templates/stork-deployment.yaml b/charts/piraeus/templates/stork-deployment.yaml
@@ -123,11 +123,12 @@ spec:
           resources:
 {{ .Values.stork.storkResources | toYaml | indent 12}}
           securityContext:
+            runAsUser: 1000
+            runAsGroup: 1000
             privileged: false
           name: stork
           env:
 {{ include "stork.config" . | indent 12 }}
-      hostPID: false
       affinity:
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -257,14 +258,16 @@ spec:
               port: 10251
             initialDelaySeconds: 15
           name: {{ .Release.Name }}-stork-scheduler
+          securityContext:
+            runAsUser: 1000
+            runAsGroup: 1000
+            privileged: false
           readinessProbe:
             httpGet:
               path: /healthz
               port: 10251
           resources:
 {{ .Values.stork.schedulerResources | toYaml | indent 12}}
-          securityContext:
-            privileged: false
       affinity:
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:

diff --git a/charts/piraeus/values.cn.yaml b/charts/piraeus/values.cn.yaml
@@ -41,6 +41,9 @@ priorityClassName: ""
 drbdRepoCred: drbdiocred # <- Specify the kubernetes secret name here
 linstorHttpsControllerSecret: "" # <- name of secret containing linstor server certificates+key. See docs/security.md
 linstorHttpsClientSecret: "" # <- name of secret containing linstor client certificates+key. See docs/security.md
+psp:
+  privilegedRole: ""
+  unprivilegedRole: ""
 operator:
   replicas: 1 # <- number of replicas for the operator deployment
   image: daocloud.io/piraeus/piraeus-operator:latest

diff --git a/charts/piraeus/values.yaml b/charts/piraeus/values.yaml
@@ -41,6 +41,9 @@ priorityClassName: ""
 drbdRepoCred: drbdiocred # <- Specify the kubernetes secret name here
 linstorHttpsControllerSecret: "" # <- name of secret containing linstor server certificates+key. See docs/security.md
 linstorHttpsClientSecret: "" # <- name of secret containing linstor client certificates+key. See docs/security.md
+psp:
+  privilegedRole: ""
+  unprivilegedRole: ""
 operator:
   replicas: 1 # <- number of replicas for the operator deployment
   image: quay.io/piraeusdatastore/piraeus-operator:latest

diff --git a/deploy/piraeus/charts/csi-snapshotter/templates/deployment.yaml b/deploy/piraeus/charts/csi-snapshotter/templates/deployment.yaml
@@ -18,6 +18,10 @@ spec:
       serviceAccountName: snapshot-controller
       containers:
         - name: snapshot-controller
+          securityContext:
+            runAsUser: 1000
+            runAsGroup: 1000
+            privileged: false
           image: quay.io/k8scsi/snapshot-controller:v2.1.0
           args:
             - "--v=5"

diff --git a/deploy/piraeus/charts/etcd/templates/rbac.yaml b/deploy/piraeus/charts/etcd/templates/rbac.yaml
@@ -0,0 +1,11 @@
+---
+# Source: piraeus/charts/etcd/templates/rbac.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: etcd
+  namespace: default
+---
+# Source: piraeus/charts/etcd/templates/rbac.yaml
+# This YAML file contains all RBAC objects that are necessary to run a
+# etcd
diff --git a/deploy/piraeus/charts/etcd/templates/statefulset.yaml b/deploy/piraeus/charts/etcd/templates/statefulset.yaml
@@ -27,6 +27,7 @@ spec:
         chart: "etcd-0.7.4"
         app: etcd
     spec:
+      serviceAccountName: "etcd"
       affinity:
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -41,6 +42,10 @@ spec:
       - name: piraeus-op-etcd
         image: "gcr.io/etcd-development/etcd:v3.4.9"
         imagePullPolicy: "IfNotPresent"
+        securityContext:
+          runAsUser: 1000
+          runAsGroup: 1000
+          privileged: false
         ports:
         - containerPort: 2380
           name: peer

diff --git a/deploy/piraeus/templates/controller-rbac.yml b/deploy/piraeus/templates/controller-rbac.yml
@@ -0,0 +1,11 @@
+---
+# Source: piraeus/templates/controller-rbac.yml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: linstor-controller
+  namespace: default
+---
+# Source: piraeus/templates/controller-rbac.yml
+# This YAML file contains all RBAC objects that are necessary to run a
+# LINSTOR controller pod
diff --git a/deploy/piraeus/templates/operator-deployment.yaml b/deploy/piraeus/templates/operator-deployment.yaml
@@ -21,6 +21,10 @@ spec:
         - name: piraeus-operator
           image: quay.io/piraeusdatastore/piraeus-operator:latest
           imagePullPolicy: "IfNotPresent"
+          securityContext:
+            runAsUser: 1000
+            runAsGroup: 1000
+            privileged: false
           env:
             - name: WATCH_NAMESPACE
               valueFrom:

diff --git a/deploy/piraeus/templates/satellite-rbac.yml b/deploy/piraeus/templates/satellite-rbac.yml
@@ -0,0 +1,11 @@
+---
+# Source: piraeus/templates/satellite-rbac.yml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: linstor-satellite
+  namespace: default
+---
+# Source: piraeus/templates/satellite-rbac.yml
+# This YAML file contains all RBAC objects that are necessary to run a
+# LINSTOR satellite pod
diff --git a/deploy/piraeus/templates/stork-deployment.yaml b/deploy/piraeus/templates/stork-deployment.yaml
@@ -207,12 +207,13 @@ spec:
           resources:
             {}
           securityContext:
+            runAsUser: 1000
+            runAsGroup: 1000
             privileged: false
           name: stork
           env:
             - name: LS_CONTROLLERS
               value: http://piraeus-op-cs.default.svc:3370
-      hostPID: false
       affinity:
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -266,14 +267,16 @@ spec:
               port: 10251
             initialDelaySeconds: 15
           name: piraeus-op-stork-scheduler
+          securityContext:
+            runAsUser: 1000
+            runAsGroup: 1000
+            privileged: false
           readinessProbe:
             httpGet:
               path: /healthz
               port: 10251
           resources:
             {}
-          securityContext:
-            privileged: false
       affinity:
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:

diff --git a/doc/helm-values.adoc b/doc/helm-values.adoc
@@ -306,7 +306,27 @@ Default:: `[]`
 Valid values:: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[tolerations]
 Description:: Tolerations to pass to the satellite pods.
 
+== PSP
 
+=== `psp.privilegedRole`
+Default:: `""`
+Valid values:: role name
+Description:: All service accounts of pods that need to run as privileged containers will be bound to this role.
++
+Currently, two components require this:
++
+- csi-nodes
+- linstor-satellites
++
+They require:
+- running as root (UID 0)
+- running as privileged containers
+- the `SYS_ADMIN` capability
+
+=== `psp.unprivilegedRole`
+Default:: `""`
+Valid values:: role name
+Description:: All service accounts of pods that do not require special privileges will be bound to this role.
 
 == Stork Scheduler
 

diff --git a/pkg/controller/linstorcontroller/linstorcontroller_controller.go b/pkg/controller/linstorcontroller/linstorcontroller_controller.go
@@ -674,14 +674,14 @@ func newDeploymentForResource(pcs *piraeusv1.LinstorController) *appsv1.Deployme
 					Labels:    labels,
 				},
 				Spec: corev1.PodSpec{
-					PriorityClassName: pcs.Spec.PriorityClassName.GetName(pcs.Namespace),
+					PriorityClassName:  pcs.Spec.PriorityClassName.GetName(pcs.Namespace),
+					ServiceAccountName: kubeSpec.LinstorControllerServiceAccount,
 					Containers: []corev1.Container{
 						{
 							Name:            "linstor-controller",
 							Image:           pcs.Spec.ControllerImage,
 							Args:            []string{"startController"}, // Run linstor-controller.
 							ImagePullPolicy: pcs.Spec.ImagePullPolicy,
-							SecurityContext: &corev1.SecurityContext{Privileged: &kubeSpec.Privileged},
 							Ports: []corev1.ContainerPort{
 								{
 									HostPort:      3376,

diff --git a/pkg/controller/linstorcsidriver/linstorcsidriver_controller.go b/pkg/controller/linstorcsidriver/linstorcsidriver_controller.go
@@ -433,9 +433,8 @@ func newCSINodeDaemonSet(csiResource *piraeusv1.LinstorCSIDriver) *appsv1.Daemon
 		Args:            []string{"--csi-endpoint=unix://$(CSI_ENDPOINT)", "--node=$(KUBE_NODE_NAME)", "--linstor-endpoint=$(LS_CONTROLLERS)", "--log-level=debug"},
 		Env:             env,
 		SecurityContext: &corev1.SecurityContext{
-			Privileged:               &IsPrivileged,
-			Capabilities:             &corev1.Capabilities{Add: []corev1.Capability{"SYS_ADMIN"}},
-			AllowPrivilegeEscalation: &IsPrivileged,
+			Privileged:   &IsPrivileged,
+			Capabilities: &corev1.Capabilities{Add: []corev1.Capability{"SYS_ADMIN"}},
 		},
 		VolumeMounts: []corev1.VolumeMount{
 			{

diff --git a/pkg/controller/linstorsatelliteset/linstorsatelliteset_controller.go b/pkg/controller/linstorsatelliteset/linstorsatelliteset_controller.go
@@ -655,11 +655,12 @@ func newDaemonSetforPNS(pns *piraeusv1.LinstorSatelliteSet, config *corev1.Confi
 					Labels:    labels,
 				},
 				Spec: corev1.PodSpec{
-					Affinity:          pns.Spec.Affinity,
-					Tolerations:       pns.Spec.Tolerations,
-					HostNetwork:       true, // INFO: Per Roland, set to true
-					DNSPolicy:         corev1.DNSClusterFirstWithHostNet,
-					PriorityClassName: pns.Spec.PriorityClassName.GetName(pns.Namespace),
+					Affinity:           pns.Spec.Affinity,
+					Tolerations:        pns.Spec.Tolerations,
+					HostNetwork:        true, // INFO: Per Roland, set to true
+					DNSPolicy:          corev1.DNSClusterFirstWithHostNet,
+					PriorityClassName:  pns.Spec.PriorityClassName.GetName(pns.Namespace),
+					ServiceAccountName: kubeSpec.LinstorSatelliteServiceAccount,
 					Containers: []corev1.Container{
 						{
 							Name:  "linstor-satellite",

diff --git a/pkg/k8s/spec/const.go b/pkg/k8s/spec/const.go
@@ -64,6 +64,8 @@ const (
 const (
 	SystemNamespace                 = "kube-system"
 	SystemCriticalPriorityClassName = "system-node-critical"
+	LinstorControllerServiceAccount = "linstor-controller"
+	LinstorSatelliteServiceAccount  = "linstor-satellite"
 )
 
 // Shared consts common to container volumes. These need to be vars, so they