improvement(k8s): add default tolerations to system services

garden-service/src/plugins/conftest/conftest.ts

@@ -20,7 +20,7 @@ import { matchGlobs, listDirectory } from "../../util/fs"
 import { PluginError } from "../../exceptions"
 import { getModuleTypeUrl, getGitHubUrl, getProviderUrl } from "../../docs/common"
 
-interface ConftestProviderConfig extends ProviderConfig {
+export interface ConftestProviderConfig extends ProviderConfig {
   policyPath: string
   namespace?: string
   testFailureThreshold: "deny" | "warn" | "none"

garden-service/src/plugins/kubernetes/config.ts

@@ -25,6 +25,7 @@ import { hotReloadableKinds, HotReloadableKind } from "./hot-reload"
 import { baseTaskSpecSchema, BaseTaskSpec, cacheResultSchema } from "../../config/task"
 import { baseTestSpecSchema, BaseTestSpec } from "../../config/test"
 import { ArtifactSpec } from "../../config/validation"
+import { V1Toleration } from "@kubernetes/client-node"
 
 export interface ProviderSecretRef {
   name: string
@@ -80,14 +81,6 @@ interface KubernetesStorage {
   sync: KubernetesStorageSpec
 }
 
-export interface Toleration {
-  effect?: "NoSchedule" | "PreferNoSchedule" | "NoExecute"
-  key?: string
-  operator: "Exists" | "Equal"
-  tolerationSeconds?: number
-  value?: string
-}
-
 export type ContainerBuildMode = "local-docker" | "cluster-docker" | "kaniko"
 
 export type DefaultDeploymentStrategy = "rolling"
@@ -110,7 +103,7 @@ export interface KubernetesConfig extends ProviderConfig {
   ingressClass?: string
   kubeconfig?: string
   namespace?: string
-  registryProxyTolerations: Toleration[]
+  registryProxyTolerations: V1Toleration[]
   resources: KubernetesResources
   storage: KubernetesStorage
   gardenSystemNamespace: string

garden-service/src/plugins/kubernetes/init.ts

@@ -34,7 +34,7 @@ import { ConfigurationError } from "../../exceptions"
 import Bluebird from "bluebird"
 import { readSecret } from "./secrets"
 import { dockerAuthSecretName, dockerAuthSecretKey } from "./constants"
-import { V1Secret } from "@kubernetes/client-node"
+import { V1Secret, V1Toleration } from "@kubernetes/client-node"
 import { KubernetesResource } from "./types"
 import { compareDeployedResources } from "./status/status"
 import { PrimitiveMap } from "../../config/common"
@@ -418,6 +418,15 @@ export function getKubernetesSystemVariables(config: KubernetesConfig) {
   const nfsStorageClass = getNfsStorageClass(config)
   const syncStorageClass = config.storage.sync.storageClass || nfsStorageClass
   const systemNamespace = config.gardenSystemNamespace
+  const systemTolerations: V1Toleration[] = [
+    {
+      key: "garden-system",
+      operator: "Equal",
+      value: "true",
+      effect: "NoSchedule",
+    },
+  ]
+  const registryProxyTolerations = config.registryProxyTolerations || systemTolerations
 
   return {
     "namespace": systemNamespace,
@@ -454,9 +463,8 @@ export function getKubernetesSystemVariables(config: KubernetesConfig) {
     "sync-storage-class": syncStorageClass,
     "sync-volume-name": `garden-sync-${syncStorageClass}`,
 
-    // Stringifying the tolerations since variable values should be primitives.
-    // Helm handles the decoding automatically.
-    "registry-proxy-tolerations": JSON.stringify(config.registryProxyTolerations),
+    "registry-proxy-tolerations": <PrimitiveMap[]>registryProxyTolerations,
+    "system-tolerations": <PrimitiveMap[]>systemTolerations,
   }
 }
 

garden-service/src/plugins/kubernetes/system.ts

@@ -21,11 +21,12 @@ import { deline, gardenAnnotationKey } from "../../util/string"
 import { deleteNamespaces } from "./namespace"
 import { PluginError } from "../../exceptions"
 import { DashboardPage } from "../../config/status"
-import { PrimitiveMap } from "../../config/common"
+import { DeepPrimitiveMap } from "../../config/common"
 import { combineStates } from "../../types/service"
 import { KubernetesResource } from "./types"
 import { defaultDotIgnoreFiles } from "../../util/fs"
 import { LogLevel } from "../../logger/log-node"
+import { ConftestProviderConfig } from "../conftest/conftest"
 
 const GARDEN_VERSION = getPackageVersion()
 const SYSTEM_NAMESPACE_MIN_VERSION = "0.9.0"
@@ -46,11 +47,18 @@ export function getSystemMetadataNamespaceName(config: KubernetesConfig) {
  */
 export async function getSystemGarden(
   ctx: KubernetesPluginContext,
-  variables: PrimitiveMap,
+  variables: DeepPrimitiveMap,
   log: LogEntry
 ): Promise<Garden> {
   const systemNamespace = await getSystemNamespace(ctx.provider, log)
 
+  const conftest: ConftestProviderConfig = {
+    environments: ["default"],
+    name: "conftest-kubernetes",
+    policyPath: "./policy",
+    testFailureThreshold: "warn",
+  }
+
   const sysProvider: KubernetesConfig = {
     ...ctx.provider.config,
     environments: ["default"],
@@ -71,7 +79,7 @@ export async function getSystemGarden(
       defaultEnvironment: "default",
       dotIgnoreFiles: defaultDotIgnoreFiles,
       environments: [{ name: "default", variables: {} }],
-      providers: [sysProvider],
+      providers: [sysProvider, conftest],
       variables,
     },
     commandInfo: ctx.command,

garden-service/src/runtime-context.ts

@@ -14,6 +14,7 @@ import { ConfigGraph, DependencyRelations } from "./config-graph"
 import { ServiceStatus } from "./types/service"
 import { RunTaskResult } from "./types/plugin/task/runTask"
 import { joiArray } from "./config/common"
+import { isPrimitive } from "util"
 
 interface RuntimeDependency {
   moduleName: string
@@ -92,9 +93,13 @@ export async function prepareRuntimeContext({
     GARDEN_VERSION: versionString,
   }
 
+  // DEPRECATED: Remove in v0.12
   for (const [key, value] of Object.entries(garden.variables)) {
-    const envVarName = `GARDEN_VARIABLES_${getEnvVarName(key)}`
-    envVars[envVarName] = value
+    // Only store primitive values, objects and arrays cause issues further down.
+    if (isPrimitive(value)) {
+      const envVarName = `GARDEN_VARIABLES_${getEnvVarName(key)}`
+      envVars[envVarName] = value
+    }
   }
 
   const result: RuntimeContext = {

garden-service/static/kubernetes/system/build-sync/garden.yml

@@ -20,3 +20,4 @@ values:
     storageClass: ${var.sync-storage-class}
   pvc:
     name: ${var.sync-volume-name}
+  tolerations: ${var.system-tolerations}

garden-service/static/kubernetes/system/docker-daemon/garden.yml

@@ -23,3 +23,4 @@ values:
   buildSync:
     volume:
       name: ${var.sync-volume-name}
+  tolerations: ${var.system-tolerations}

garden-service/static/kubernetes/system/docker-registry/garden.yml

@@ -26,3 +26,4 @@ values:
     size: ${var.registry-storage-size}
     storageClass: ${var.registry-storage-class}
     deleteEnabled: true
+  tolerations: ${var.system-tolerations}

garden-service/static/kubernetes/system/ingress-controller/garden.yml

@@ -24,5 +24,6 @@ values:
     service:
       omitClusterIP: true
     minReadySeconds: 1
+    tolerations: ${var.system-tolerations}
   defaultBackend:
     enabled: false

garden-service/static/kubernetes/system/nfs-provisioner/garden.yml

@@ -12,3 +12,4 @@ values:
     storageClass: ${var.nfs-storage-class}
   storageClass:
     name: ${var.sync-storage-class}
+  tolerations: ${var.system-tolerations}

garden-service/static/kubernetes/system/policy/base_test.rego

@@ -0,0 +1,31 @@
+package main
+
+empty(value) {
+  count(value) == 0
+}
+
+no_violations {
+  empty(deny)
+}
+
+no_warnings {
+  empty(warn)
+}
+
+test_deployment_without_security_context {
+  deny["Containers must not run as root in Deployment sample"] with input as {"kind": "Deployment", "metadata": { "name": "sample" }}
+}
+
+test_deployment_with_security_context {
+  no_violations with input as {"kind": "Deployment", "metadata": {"name": "sample"}, "spec": {
+    "selector": { "matchLabels": { "app": "something", "release": "something" }},
+    "template": { "spec": { "securityContext": { "runAsNonRoot": true  }}}}}
+}
+
+test_services_not_denied {
+  no_violations with input as {"kind": "Service", "metadata": { "name": "sample" }}
+}
+
+test_services_issue_warning {
+  warn["Found service sample but services are not allowed"] with input as {"kind": "Service", "metadata": { "name": "sample" }}
+}

garden-service/static/kubernetes/system/policy/deny.rego

@@ -0,0 +1,18 @@
+package main
+
+import data.kubernetes
+
+name = input.metadata.name
+
+deny[msg] {
+  kubernetes.is_deployment
+  toleration := {
+    "key": "garden-system",
+    "operator": "Equal",
+    "value": "true",
+    "effect": "NoSchedule",
+  }
+  input.spec.template.spec.tolerations[_] != toleration
+
+  msg = sprintf("Deployment %s is missing toleration of kind %v", [name, toleration])
+}

garden-service/static/kubernetes/system/policy/kubernetes.rego

@@ -0,0 +1,9 @@
+package kubernetes
+
+is_service {
+  input.kind = "Service"
+}
+
+is_deployment {
+  input.kind = "Deployment"
+}

garden-service/static/kubernetes/system/policy/labels.rego

@@ -0,0 +1,21 @@
+package main
+
+import data.kubernetes
+
+name = input.metadata.name
+
+# TODO: Re-enable this policy (or some version thereof)
+# labels {
+#     input.metadata.labels["app.kubernetes.io/name"]
+#     input.metadata.labels["app.kubernetes.io/instance"]
+#     input.metadata.labels["app.kubernetes.io/version"]
+#     input.metadata.labels["app.kubernetes.io/component"]
+#     input.metadata.labels["app.kubernetes.io/part-of"]
+#     input.metadata.labels["app.kubernetes.io/managed-by"]
+# }
+#
+# deny[msg] {
+#   kubernetes.is_deployment
+#   not labels
+#   msg = sprintf("%s must include Kubernetes recommended labels: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/#labels ", [name])
+# }

garden-service/static/kubernetes/system/policy/warn.rego

@@ -0,0 +1,6 @@
+package main
+
+import data.kubernetes
+
+name = input.metadata.name
+

garden-service/static/kubernetes/system/registry-proxy/templates/daemonset.yaml

@@ -51,7 +51,7 @@ spec:
     {{- with .Values.tolerations }}
       # Note that we're not using the toYAML function here because the tolerations are JSON stringified (by Garden)
       tolerations:
-        {{- . | nindent 8 }}
+        {{- toYaml . | nindent 8 }}
     {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:

garden-service/test/integ/src/plugins/kubernetes/system.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2018-2020 Garden Technologies, Inc. <[email protected]>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import { Garden } from "../../../../../src/garden"
+import { Provider } from "../../../../../src/config/provider"
+import { KubernetesConfig, KubernetesPluginContext } from "../../../../../src/plugins/kubernetes/config"
+import { getDataDir, makeTestGarden } from "../../../../helpers"
+import { expect } from "chai"
+import { TestTask } from "../../../../../src/tasks/test"
+import { getSystemGarden } from "../../../../../src/plugins/kubernetes/system"
+import { getKubernetesSystemVariables } from "../../../../../src/plugins/kubernetes/init"
+import Bluebird = require("bluebird")
+
+describe("System services", () => {
+  let garden: Garden
+  let provider: Provider<KubernetesConfig>
+
+  before(async () => {
+    const root = getDataDir("test-projects", "container")
+    garden = await makeTestGarden(root)
+    provider = (await garden.resolveProvider("local-kubernetes")) as Provider<KubernetesConfig>
+  })
+
+  after(async () => {
+    await garden.close()
+  })
+
+  it("should use conftest to check whether system services have a valid config", async () => {
+    const ctx = <KubernetesPluginContext>garden.getPluginContext(provider)
+    const variables = getKubernetesSystemVariables(provider.config)
+    const systemGarden = await getSystemGarden(ctx, variables, garden.log)
+    const graph = await systemGarden.getConfigGraph(garden.log)
+    const modules = (await graph.getModules()).filter((module) => module.name.startsWith("conftest-"))
+
+    await Bluebird.map(modules, async (module) => {
+      const testTask = new TestTask({
+        garden: systemGarden,
+        module,
+        log: garden.log,
+        graph,
+        testConfig: module.testConfigs[0] || {},
+        force: true,
+        forceBuild: true,
+        version: module.version,
+        _guard: true,
+      })
+      const key = testTask.getKey()
+      const result = await systemGarden.processTasks([testTask])
+      expect(result[key]).to.exist
+      expect(result[key]?.error).to.not.exist
+    })
+  })
+})