From 8bd67d6fb4daf51fade8a8f67b0dbb8d30dc8817 Mon Sep 17 00:00:00 2001 From: Matt Schallert Date: Fri, 22 Mar 2019 17:03:48 -0400 Subject: [PATCH] [api] allow specifying security contexts (#107) Allows users to specify their own pod and container-level security contexts for all pods created by the operator's statefulsets. Also makes the operator's pod itself a more locked down default. --- Makefile | 9 ++- bundle.yaml | 4 ++ docs/api.md | 2 + .../m3db-operator/templates/stateful_set.yaml | 4 ++ pkg/apis/m3dboperator/v1alpha1/cluster.go | 8 +++ .../v1alpha1/zz_generated.deepcopy.go | 10 ++++ pkg/k8sops/fixtures/testM3DBCluster.yaml | 4 ++ pkg/k8sops/generators.go | 30 ---------- pkg/k8sops/generators_test.go | 16 +++--- pkg/k8sops/statefulset.go | 56 +++++++++++-------- 10 files changed, 77 insertions(+), 66 deletions(-) diff --git a/Makefile b/Makefile index 0c544e2d..74a2ea41 100644 --- a/Makefile +++ b/Makefile @@ -241,12 +241,15 @@ build-docker: ## Build m3db-operator docker image with go binary @echo "--- $@" @./build/build-docker.sh -.PHONE: helm-bundle -helm-bundle: install-codegen-tools +.PHONY: helm-bundle-no-deps +helm-bundle-no-deps: @echo "--- $@" - @helm template helm/m3db-operator > bundle.yaml + @helm template --namespace default helm/m3db-operator > bundle.yaml @PATH=$(retool_bin_path):$(PATH) kubeval -v=1.12.0 bundle.yaml +.PHONY: helm-bundle +helm-bundle: install-codegen-tools helm-bundle-no-deps + .PHONY: publish-helm-charts publish-helm-charts: ## pushes a new version of the helm chart @echo "+ $@" diff --git a/bundle.yaml b/bundle.yaml index c5df0fb1..0441a322 100644 --- a/bundle.yaml +++ b/bundle.yaml @@ -77,6 +77,10 @@ spec: labels: name: m3db-operator spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 containers: - name: m3db-operator image: quay.io/m3db/m3db-operator:v0.1.4 diff --git a/docs/api.md b/docs/api.md index 2c670cbc..5fdd1e9a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -48,6 +48,8 @@ ClusterSpec defines the desired state for a M3 cluster to be converge to. | podIdentityConfig | PodIdentityConfig sets the configuration for pod identity. If unset only pod name and UID will be used. | *PodIdentityConfig | false | | containerResources | Resources defines memory / cpu constraints for each container in the cluster. | [corev1.ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#resourcerequirements-v1-core) | false | | dataDirVolumeClaimTemplate | DataDirVolumeClaimTemplate is the volume claim template for an M3DB instance's data. It claims PersistentVolumes for cluster storage, volumes are dynamically provisioned by when the StorageClass is defined. | *[corev1.PersistentVolumeClaim](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#persistentvolumeclaim-v1-core) | false | +| podSecurityContext | PodSecurityContext allows the user to specify an optional security context for pods. | *corev1.PodSecurityContext | false | +| securityContext | SecurityContext allows the user to specify a container-level security context. | *corev1.SecurityContext | false | | labels | Labels sets the base labels that will be applied to resources created by the cluster. // TODO(schallert): design doc on labeling scheme. | map[string]string | false | [Back to TOC](#table-of-contents) diff --git a/helm/m3db-operator/templates/stateful_set.yaml b/helm/m3db-operator/templates/stateful_set.yaml index d1002378..e4ed3366 100644 --- a/helm/m3db-operator/templates/stateful_set.yaml +++ b/helm/m3db-operator/templates/stateful_set.yaml @@ -14,6 +14,10 @@ spec: labels: name: {{ .Values.operator.name }} spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 containers: - name: {{ .Values.operator.name }} image: {{ .Values.image.repository}}:{{ .Values.image.tag }} diff --git a/pkg/apis/m3dboperator/v1alpha1/cluster.go b/pkg/apis/m3dboperator/v1alpha1/cluster.go index 703e114a..1b68d683 100644 --- a/pkg/apis/m3dboperator/v1alpha1/cluster.go +++ b/pkg/apis/m3dboperator/v1alpha1/cluster.go @@ -212,6 +212,14 @@ type ClusterSpec struct { // +optional DataDirVolumeClaimTemplate *corev1.PersistentVolumeClaim `json:"dataDirVolumeClaimTemplate,omitempty" yaml:"dataDirVolumeClaimTemplate"` + // PodSecurityContext allows the user to specify an optional security context + // for pods. + PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` + + // SecurityContext allows the user to specify a container-level security + // context. + SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` + // Labels sets the base labels that will be applied to resources created by // the cluster. // TODO(schallert): design doc on labeling scheme. Labels map[string]string `json:"labels,omitempty" yaml:"labels"` diff --git a/pkg/apis/m3dboperator/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/m3dboperator/v1alpha1/zz_generated.deepcopy.go index bcc90491..98414cec 100644 --- a/pkg/apis/m3dboperator/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/m3dboperator/v1alpha1/zz_generated.deepcopy.go @@ -83,6 +83,16 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(v1.PersistentVolumeClaim) (*in).DeepCopyInto(*out) } + if in.PodSecurityContext != nil { + in, out := &in.PodSecurityContext, &out.PodSecurityContext + *out = new(v1.PodSecurityContext) + (*in).DeepCopyInto(*out) + } + if in.SecurityContext != nil { + in, out := &in.SecurityContext, &out.SecurityContext + *out = new(v1.SecurityContext) + (*in).DeepCopyInto(*out) + } if in.Labels != nil { in, out := &in.Labels, &out.Labels *out = make(map[string]string, len(*in)) diff --git a/pkg/k8sops/fixtures/testM3DBCluster.yaml b/pkg/k8sops/fixtures/testM3DBCluster.yaml index 46e439c5..64523691 100644 --- a/pkg/k8sops/fixtures/testM3DBCluster.yaml +++ b/pkg/k8sops/fixtures/testM3DBCluster.yaml @@ -25,6 +25,10 @@ spec: limits: memory: 2Gi cpu: '2' + podSecurityContext: + fsGroup: 10 + securityContext: + runAsUser: 20 etcdEndpoints: - ep0 - ep1 diff --git a/pkg/k8sops/generators.go b/pkg/k8sops/generators.go index 48557a72..3ca0032a 100644 --- a/pkg/k8sops/generators.go +++ b/pkg/k8sops/generators.go @@ -32,7 +32,6 @@ import ( v1 "k8s.io/api/core/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" "github.com/kubernetes/utils/pointer" ) @@ -132,37 +131,8 @@ func GenerateStatefulSet( clusterSpec := cluster.Spec - // TODO(schallert): we're currently using the health of the coordinator for - // liveness probes until https://github.com/m3db/m3/issues/996 is fixed. Move - // to the dbnode's health endpoint once fixed. - probeHealth := &v1.Probe{ - TimeoutSeconds: _probeTimeoutSeconds, - InitialDelaySeconds: _probeInitialDelaySeconds, - FailureThreshold: _probeFailureThreshold, - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Port: intstr.FromInt(_probePort), - Path: _probePathHealth, - Scheme: v1.URISchemeHTTP, - }, - }, - } - - probeReady := &v1.Probe{ - TimeoutSeconds: _probeTimeoutSeconds, - InitialDelaySeconds: _probeInitialDelaySeconds, - FailureThreshold: _probeFailureThreshold, - Handler: v1.Handler{ - Exec: &v1.ExecAction{ - Command: []string{_healthFileName}, - }, - }, - } - statefulSet := NewBaseStatefulSet(ssName, isolationGroupName, cluster, instanceAmount) m3dbContainer := &statefulSet.Spec.Template.Spec.Containers[0] - m3dbContainer.LivenessProbe = probeHealth - m3dbContainer.ReadinessProbe = probeReady m3dbContainer.Resources = clusterSpec.ContainerResources m3dbContainer.Ports = generateContainerPorts() statefulSet.Spec.Template.Spec.Affinity = GenerateStatefulSetAffinity(isolationGroup) diff --git a/pkg/k8sops/generators_test.go b/pkg/k8sops/generators_test.go index 20803be3..490a33c6 100644 --- a/pkg/k8sops/generators_test.go +++ b/pkg/k8sops/generators_test.go @@ -132,6 +132,9 @@ func TestGenerateStatefulSet(t *testing.T) { Labels: labels, }, Spec: v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + FSGroup: pointer.Int64Ptr(10), + }, Affinity: &v1.Affinity{ NodeAffinity: &v1.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ @@ -151,17 +154,12 @@ func TestGenerateStatefulSet(t *testing.T) { }, Containers: []v1.Container{ { - Name: ssName, - SecurityContext: &v1.SecurityContext{ - Privileged: &[]bool{true}[0], - Capabilities: &v1.Capabilities{ - Add: []v1.Capability{ - "IPC_LOCK", - }, - }, - }, + Name: ssName, LivenessProbe: health, ReadinessProbe: readiness, + SecurityContext: &v1.SecurityContext{ + RunAsUser: pointer.Int64Ptr(20), + }, Command: []string{ "m3dbnode", }, diff --git a/pkg/k8sops/statefulset.go b/pkg/k8sops/statefulset.go index 8627fe0e..282497b2 100644 --- a/pkg/k8sops/statefulset.go +++ b/pkg/k8sops/statefulset.go @@ -193,20 +193,6 @@ func (k *k8sops) CheckStatefulStatus(cluster *myspec.M3DBCluster, statefulSet *a } // NewBaseProbe returns a probe configured for default ports. -func NewBaseProbe() *v1.Probe { - return &v1.Probe{ - TimeoutSeconds: _probeTimeoutSeconds, - InitialDelaySeconds: _probeInitialDelaySeconds, - FailureThreshold: _probeFailureThreshold, - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Port: intstr.FromInt(_probePort), - Path: _probePathHealth, - Scheme: v1.URISchemeHTTP, - }, - }, - } -} // NewBaseStatefulSet returns a base configured stateful set. func NewBaseStatefulSet(ssName, isolationGroup string, cluster *myspec.M3DBCluster, instanceCount int32) *appsv1.StatefulSet { @@ -223,6 +209,33 @@ func NewBaseStatefulSet(ssName, isolationGroup string, cluster *myspec.M3DBClust objLabels[k] = v } + // TODO(schallert): we're currently using the health of the coordinator for + // liveness probes until https://github.com/m3db/m3/issues/996 is fixed. Move + // to the dbnode's health endpoint once fixed. + probeHealth := &v1.Probe{ + TimeoutSeconds: _probeTimeoutSeconds, + InitialDelaySeconds: _probeInitialDelaySeconds, + FailureThreshold: _probeFailureThreshold, + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Port: intstr.FromInt(_probePort), + Path: _probePathHealth, + Scheme: v1.URISchemeHTTP, + }, + }, + } + + probeReady := &v1.Probe{ + TimeoutSeconds: _probeTimeoutSeconds, + InitialDelaySeconds: _probeInitialDelaySeconds, + FailureThreshold: _probeFailureThreshold, + Handler: v1.Handler{ + Exec: &v1.ExecAction{ + Command: []string{_healthFileName}, + }, + }, + } + return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: ssName, @@ -239,18 +252,13 @@ func NewBaseStatefulSet(ssName, isolationGroup string, cluster *myspec.M3DBClust Labels: objLabels, }, Spec: v1.PodSpec{ + SecurityContext: cluster.Spec.PodSecurityContext, Containers: []v1.Container{ { - Name: ssName, - SecurityContext: &v1.SecurityContext{ - Privileged: &[]bool{true}[0], - Capabilities: &v1.Capabilities{ - Add: []v1.Capability{ - "IPC_LOCK", - }, - }, - }, - ReadinessProbe: nil, + Name: ssName, + SecurityContext: cluster.Spec.SecurityContext, + ReadinessProbe: probeReady, + LivenessProbe: probeHealth, Command: []string{ "m3dbnode", },