From 3546842360af1fc7b1d21dbcaf1623febe36aa79 Mon Sep 17 00:00:00 2001 From: pidah Date: Thu, 18 Jan 2018 00:45:49 +0000 Subject: [PATCH] Add Enable Pod Security Option (#2048) * Add PodSecurityPolicy * use helpers.IsTrueBoolPointer, delete EnablePodSecurityPolicy function and update defaultAPIServerConfig --- parts/k8s/kubernetesmastercustomscript.sh | 11 ++ .../kubernetesmaster-pod-security-policy.yaml | 128 ++++++++++++++++++ pkg/acsengine/addons.go | 6 + pkg/acsengine/const.go | 2 + pkg/acsengine/defaults-apiserver.go | 5 + pkg/acsengine/defaults-controller-manager.go | 9 +- pkg/api/converterfromapi.go | 1 + pkg/api/convertertoapi.go | 1 + pkg/api/types.go | 1 + pkg/api/vlabs/types.go | 1 + pkg/api/vlabs/validate.go | 4 + 11 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 parts/k8s/manifests/kubernetesmaster-pod-security-policy.yaml diff --git a/parts/k8s/kubernetesmastercustomscript.sh b/parts/k8s/kubernetesmastercustomscript.sh index 4689b1155d..64096593d2 100644 --- a/parts/k8s/kubernetesmastercustomscript.sh +++ b/parts/k8s/kubernetesmastercustomscript.sh @@ -607,6 +607,16 @@ function ensureEtcdDataDir() { exit 4 } +function ensurePodSecurityPolicy(){ + if $REBOOTREQUIRED; then + return + fi + POD_SECURITY_POLICY_FILE="/etc/kubernetes/manifests/pod-security-policy.yaml" + if [ -f $POD_SECURITY_POLICY_FILE ]; then + kubectl create -f $POD_SECURITY_POLICY_FILE + fi +} + function writeKubeConfig() { KUBECONFIGDIR=/home/$ADMINUSER/.kube KUBECONFIGFILE=$KUBECONFIGDIR/config @@ -695,6 +705,7 @@ if [[ ! -z "${APISERVER_PRIVATE_KEY}" ]]; then ensureEtcdDataDir ensureEtcd ensureApiserver + ensurePodSecurityPolicy fi if [[ $OS == $UBUNTU_OS_NAME ]]; then diff --git a/parts/k8s/manifests/kubernetesmaster-pod-security-policy.yaml b/parts/k8s/manifests/kubernetesmaster-pod-security-policy.yaml new file mode 100644 index 0000000000..bb2ca89ac7 --- /dev/null +++ b/parts/k8s/manifests/kubernetesmaster-pod-security-policy.yaml @@ -0,0 +1,128 @@ +apiVersion: extensions/v1beta1 +kind: PodSecurityPolicy +metadata: + name: privileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: "*" +spec: + privileged: true + allowPrivilegeEscalation: true + allowedCapabilities: + - "*" + volumes: + - "*" + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + hostIPC: true + hostPID: true + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' +--- +apiVersion: extensions/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default' + apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' +spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + # This is redundant with non-root + disallow privilege escalation, + # but we can provide it for defense in depth. + requiredDropCapabilities: + - ALL + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + # Assume that persistentVolumes set up by the cluster admin are safe to use. + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'MustRunAsNonRoot' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:privileged +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - privileged +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:restricted +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - restricted +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default:restricted +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:restricted +subjects: +- kind: Group + name: system:authenticated + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: default:privileged + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:privileged +subjects: +- kind: Group + name: system:masters + apiGroup: rbac.authorization.k8s.io +- kind: Group + name: system:serviceaccounts:kube-system + apiGroup: rbac.authorization.k8s.io diff --git a/pkg/acsengine/addons.go b/pkg/acsengine/addons.go index bcd3538562..ec6429c4ab 100644 --- a/pkg/acsengine/addons.go +++ b/pkg/acsengine/addons.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/Azure/acs-engine/pkg/api" + "github.com/Azure/acs-engine/pkg/helpers" ) type kubernetesFeatureSetting struct { @@ -85,6 +86,11 @@ func kubernetesManifestSettingsInit(profile *api.Properties) []kubernetesFeature "cloud-controller-manager.yaml", profile.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager != nil && *profile.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager, }, + { + "kubernetesmaster-pod-security-policy.yaml", + "pod-security-policy.yaml", + helpers.IsTrueBoolPointer(profile.OrchestratorProfile.KubernetesConfig.EnablePodSecurityPolicy), + }, { "kubernetesmaster-kube-apiserver.yaml", "kube-apiserver.yaml", diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index 0c42619f52..518921a1a7 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -58,6 +58,8 @@ const ( DefaultKubernetesCtrlMgrRouteReconciliationPeriod = "10s" // DefaultKubernetesCtrlMgrTerminatedPodGcThreshold is set to 5000, see --terminated-pod-gc-threshold at https://kubernetes.io/docs/admin/kube-controller-manager/ and https://github.com/kubernetes/kubernetes/issues/22680 DefaultKubernetesCtrlMgrTerminatedPodGcThreshold = "5000" + // DefaultKubernetesCtrlMgrUseSvcAccountCreds is "true", see --use-service-account-credentials at https://kubernetes.io/docs/admin/kube-controller-manager/ + DefaultKubernetesCtrlMgrUseSvcAccountCreds = "true" // DefaultKubernetesCloudProviderBackoff is false to disable cloudprovider backoff implementation for API calls DefaultKubernetesCloudProviderBackoff = false // DefaultKubernetesCloudProviderBackoffRetries is 6, takes effect if DefaultKubernetesCloudProviderBackoff is true diff --git a/pkg/acsengine/defaults-apiserver.go b/pkg/acsengine/defaults-apiserver.go index 3e861c67e0..c6e6fbcca6 100644 --- a/pkg/acsengine/defaults-apiserver.go +++ b/pkg/acsengine/defaults-apiserver.go @@ -99,6 +99,11 @@ func setAPIServerConfig(cs *api.ContainerService) { } } + // Pod Security Policy configuration + if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnablePodSecurityPolicy) { + defaultAPIServerConfig["--admission-control"] = defaultAPIServerConfig["--admission-control"] + ",PodSecurityPolicy" + } + // If no user-configurable apiserver config values exists, use the defaults if o.KubernetesConfig.APIServerConfig == nil { o.KubernetesConfig.APIServerConfig = defaultAPIServerConfig diff --git a/pkg/acsengine/defaults-controller-manager.go b/pkg/acsengine/defaults-controller-manager.go index 5e9053bb8d..156bc3f7db 100644 --- a/pkg/acsengine/defaults-controller-manager.go +++ b/pkg/acsengine/defaults-controller-manager.go @@ -44,10 +44,11 @@ func setControllerManagerConfig(cs *api.ContainerService) { // Default controller-manager config defaultControllerManagerConfig := map[string]string{ - "--node-monitor-grace-period": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, - "--pod-eviction-timeout": DefaultKubernetesCtrlMgrPodEvictionTimeout, - "--route-reconciliation-period": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "--terminated-pod-gc-threshold": DefaultKubernetesCtrlMgrTerminatedPodGcThreshold, + "--node-monitor-grace-period": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, + "--pod-eviction-timeout": DefaultKubernetesCtrlMgrPodEvictionTimeout, + "--route-reconciliation-period": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, + "--terminated-pod-gc-threshold": DefaultKubernetesCtrlMgrTerminatedPodGcThreshold, + "--use-service-account-credentials": DefaultKubernetesCtrlMgrUseSvcAccountCreds, } // If no user-configurable controller-manager config values exists, use the defaults diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index b5a3027fb5..7716386788 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -669,6 +669,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet vlabs.EnableSecureKubelet = api.EnableSecureKubelet vlabs.EnableAggregatedAPIs = api.EnableAggregatedAPIs vlabs.EnableDataEncryptionAtRest = api.EnableDataEncryptionAtRest + vlabs.EnablePodSecurityPolicy = api.EnablePodSecurityPolicy vlabs.GCHighThreshold = api.GCHighThreshold vlabs.GCLowThreshold = api.GCLowThreshold vlabs.EtcdVersion = api.EtcdVersion diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index 025a4f22f8..2bf89ae1ec 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -614,6 +614,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes api.EnableSecureKubelet = vlabs.EnableSecureKubelet api.EnableAggregatedAPIs = vlabs.EnableAggregatedAPIs api.EnableDataEncryptionAtRest = vlabs.EnableDataEncryptionAtRest + api.EnablePodSecurityPolicy = vlabs.EnablePodSecurityPolicy api.GCHighThreshold = vlabs.GCHighThreshold api.GCLowThreshold = vlabs.GCLowThreshold api.EtcdVersion = vlabs.EtcdVersion diff --git a/pkg/api/types.go b/pkg/api/types.go index 4a137a55da..9cad5a0e6d 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -243,6 +243,7 @@ type KubernetesConfig struct { EtcdVersion string `json:"etcdVersion,omitempty"` EtcdDiskSizeGB string `json:"etcdDiskSizeGB,omitempty"` EnableDataEncryptionAtRest *bool `json:"enableDataEncryptionAtRest,omitempty"` + EnablePodSecurityPolicy *bool `json:"enablePodSecurityPolicy,omitempty"` Addons []KubernetesAddon `json:"addons,omitempty"` KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index ac1b66b9e7..6832382a58 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -251,6 +251,7 @@ type KubernetesConfig struct { EtcdVersion string `json:"etcdVersion,omitempty"` EtcdDiskSizeGB string `json:"etcdDiskSizeGB,omitempty"` EnableDataEncryptionAtRest *bool `json:"enableDataEncryptionAtRest,omitempty"` + EnablePodSecurityPolicy *bool `json:"enablePodSecurityPolicy,omitempty"` Addons []KubernetesAddon `json:"addons,omitempty"` KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 0bcbb2bbf5..c4babe1265 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -111,6 +111,10 @@ func (o *OrchestratorProfile) Validate(isUpdate bool) error { } } } + if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnablePodSecurityPolicy) && + !helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableRbac) { + return fmt.Errorf("enablePodSecurityPolicy requires the enableRbac feature as a prerequisite") + } } default: