feat(k8s): expose nodeSelector field on system PodSpecs

garden-io · Mar 30, 2020 · 9939014 · 9939014
1 parent 63f0a04
commit 9939014
Show file tree

Hide file tree

Showing 14 changed files with 90 additions and 3 deletions.
diff --git a/docs/reference/providers/kubernetes.md b/docs/reference/providers/kubernetes.md
@@ -238,6 +238,12 @@ providers:
       # for now).
       acmeChallengeType: HTTP-01
 
+    # Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain
+    # the system services to only run on particular nodes. [See
+    # here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to
+    # assigning Pods to nodes.
+    systemNodeSelector: {}
+
     # For setting tolerations on the registry-proxy when using in-cluster building.
     # The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node.
     #
@@ -1204,6 +1210,25 @@ providers:
       acmeChallengeType: "HTTP-01"
 ```
 
+### `providers[].systemNodeSelector`
+
+[providers](#providers) > systemNodeSelector
+
+Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain
+the system services to only run on particular nodes. [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
+
+| Type     | Default | Required |
+| -------- | ------- | -------- |
+| `object` | `{}`    | No       |
+
+Example:
+
+```yaml
+providers:
+  - systemNodeSelector:
+        disktype: ssd
+```
+
 ### `providers[].registryProxyTolerations[]`
 
 [providers](#providers) > registryProxyTolerations

diff --git a/docs/reference/providers/local-kubernetes.md b/docs/reference/providers/local-kubernetes.md
@@ -234,6 +234,12 @@ providers:
       # for now).
       acmeChallengeType: HTTP-01
 
+    # Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain
+    # the system services to only run on particular nodes. [See
+    # here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to
+    # assigning Pods to nodes.
+    systemNodeSelector: {}
+
     # For setting tolerations on the registry-proxy when using in-cluster building.
     # The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node.
     #
@@ -1172,6 +1178,25 @@ providers:
       acmeChallengeType: "HTTP-01"
 ```
 
+### `providers[].systemNodeSelector`
+
+[providers](#providers) > systemNodeSelector
+
+Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain
+the system services to only run on particular nodes. [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
+
+| Type     | Default | Required |
+| -------- | ------- | -------- |
+| `object` | `{}`    | No       |
+
+Example:
+
+```yaml
+providers:
+  - systemNodeSelector:
+        disktype: ssd
+```
+
 ### `providers[].registryProxyTolerations[]`
 
 [providers](#providers) > registryProxyTolerations

diff --git a/garden-service/src/plugins/kubernetes/config.ts b/garden-service/src/plugins/kubernetes/config.ts
@@ -8,7 +8,7 @@
 
 import dedent = require("dedent")
 
-import { joiArray, joiIdentifier, joiProviderName, joi } from "../../config/common"
+import { joiArray, joiIdentifier, joiProviderName, joi, joiStringMap } from "../../config/common"
 import { Provider, providerConfigBaseSchema, ProviderConfig } from "../../config/provider"
 import {
   containerRegistryConfigSchema,
@@ -104,6 +104,7 @@ export interface KubernetesConfig extends ProviderConfig {
   kubeconfig?: string
   namespace?: string
   registryProxyTolerations: V1Toleration[]
+  systemNodeSelector: { [key: string]: string }
   resources: KubernetesResources
   storage: KubernetesStorage
   gardenSystemNamespace: string
@@ -484,6 +485,15 @@ export const kubernetesConfigBase = providerConfigBaseSchema().keys({
     }).description(dedent`cert-manager configuration, for creating and managing TLS certificates. See the
         [cert-manager guide](https://docs.garden.io/advanced/cert-manager-integration) for details.`),
   _systemServices: joiArray(joiIdentifier()).meta({ internal: true }),
+  systemNodeSelector: joiStringMap(joi.string())
+    .description(
+      dedent`
+      Exposes the \`nodeSelector\` field on the PodSpec of system services. This allows you to constrain
+      the system services to only run on particular nodes. [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
+    `
+    )
+    .example({ disktype: "ssd" })
+    .default(() => ({})),
   registryProxyTolerations: joiArray(
     joi.object().keys({
       effect: joi.string().allow("NoSchedule", "PreferNoSchedule", "NoExecute").description(dedent`

diff --git a/garden-service/src/plugins/kubernetes/init.ts b/garden-service/src/plugins/kubernetes/init.ts
@@ -465,6 +465,7 @@ export function getKubernetesSystemVariables(config: KubernetesConfig) {
 
     "registry-proxy-tolerations": <PrimitiveMap[]>registryProxyTolerations,
     "system-tolerations": <PrimitiveMap[]>systemTolerations,
+    "system-node-selector": config.systemNodeSelector,
   }
 }
 

diff --git a/garden-service/src/plugins/kubernetes/system.ts b/garden-service/src/plugins/kubernetes/system.ts
@@ -55,7 +55,7 @@ export async function getSystemGarden(
   const conftest: ConftestProviderConfig = {
     environments: ["default"],
     name: "conftest-kubernetes",
-    policyPath: "./policy",
+    policyPath: "policy",
     testFailureThreshold: "warn",
   }
 

diff --git a/garden-service/static/kubernetes/system/build-sync/garden.yml b/garden-service/static/kubernetes/system/build-sync/garden.yml
@@ -21,3 +21,4 @@ values:
   pvc:
     name: ${var.sync-volume-name}
   tolerations: ${var.system-tolerations}
+  nodeSelector: ${var.system-node-selector}
diff --git a/garden-service/static/kubernetes/system/docker-daemon/garden.yml b/garden-service/static/kubernetes/system/docker-daemon/garden.yml
@@ -23,4 +23,5 @@ values:
   buildSync:
     volume:
       name: ${var.sync-volume-name}
-  tolerations: ${var.system-tolerations}
+  tolerations: ${var.system-tolerations}
+  nodeSelector: ${var.system-node-selector}
diff --git a/garden-service/static/kubernetes/system/docker-registry/garden.yml b/garden-service/static/kubernetes/system/docker-registry/garden.yml
@@ -27,3 +27,4 @@ values:
     storageClass: ${var.registry-storage-class}
     deleteEnabled: true
   tolerations: ${var.system-tolerations}
+  nodeSelector: ${var.system-node-selector}
diff --git a/garden-service/static/kubernetes/system/ingress-controller/garden.yml b/garden-service/static/kubernetes/system/ingress-controller/garden.yml
@@ -25,5 +25,6 @@ values:
       omitClusterIP: true
     minReadySeconds: 1
     tolerations: ${var.system-tolerations}
+    nodeSelector: ${var.system-node-selector}
   defaultBackend:
     enabled: false
diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/garden.yml b/garden-service/static/kubernetes/system/nfs-provisioner/garden.yml
@@ -13,3 +13,4 @@ values:
   storageClass:
     name: ${var.sync-storage-class}
   tolerations: ${var.system-tolerations}
+  nodeSelector: ${var.system-node-selector}
diff --git a/garden-service/test/integ/src/plugins/kubernetes/system.ts b/garden-service/test/integ/src/plugins/kubernetes/system.ts
@@ -31,6 +31,24 @@ describe("System services", () => {
   })
 
   it("should use conftest to check whether system services have a valid config", async () => {
+    const ctx = <KubernetesPluginContext>garden.getPluginContext(provider)
+    const variables = getKubernetesSystemVariables(provider.config)
+    const systemGarden = await getSystemGarden(ctx, variables, garden.log)
+    const graph = await systemGarden.getConfigGraph(garden.log)
+    const conftestModuleNames = (await graph.getModules())
+      .filter((module) => module.name.startsWith("conftest-"))
+      .map((m) => m.name)
+    expect(conftestModuleNames.sort()).to.eql([
+      "conftest-build-sync",
+      "conftest-docker-daemon",
+      "conftest-docker-registry",
+      "conftest-ingress-controller",
+      "conftest-nfs-provisioner",
+      "conftest-registry-proxy",
+    ])
+  })
+
+  it("should check whether system modules pass the conftest test", async () => {
     const ctx = <KubernetesPluginContext>garden.getPluginContext(provider)
     const variables = getKubernetesSystemVariables(provider.config)
     const systemGarden = await getSystemGarden(ctx, variables, garden.log)

diff --git a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts
@@ -60,6 +60,7 @@ const basicConfig: KubernetesConfig = {
   ingressHttpsPort: 443,
   resources: defaultResources,
   storage: defaultStorage,
+  systemNodeSelector: {},
   registryProxyTolerations: [],
   tlsCertificates: [],
   _systemServices: [],

diff --git a/garden-service/test/unit/src/plugins/kubernetes/init.ts b/garden-service/test/unit/src/plugins/kubernetes/init.ts
@@ -53,6 +53,7 @@ const basicConfig: KubernetesConfig = {
   ingressHttpsPort: 443,
   resources: defaultResources,
   storage: defaultStorage,
+  systemNodeSelector: {},
   registryProxyTolerations: [],
   tlsCertificates: [],
   _systemServices: [],

diff --git a/garden-service/test/unit/src/plugins/kubernetes/kubernetes.ts b/garden-service/test/unit/src/plugins/kubernetes/kubernetes.ts
@@ -27,6 +27,7 @@ describe("kubernetes configureProvider", () => {
     ingressHttpsPort: 443,
     resources: defaultResources,
     storage: defaultStorage,
+    systemNodeSelector: {},
     registryProxyTolerations: [],
     tlsCertificates: [],
     _systemServices: [],
-Original file line number
+Diff line change
@@ Expand Up @@
         "registry-proxy-tolerations": <PrimitiveMap[]>registryProxyTolerations,
         "system-tolerations": <PrimitiveMap[]>systemTolerations,
+        "system-node-selector": config.systemNodeSelector,
       }
     }
@@ Expand Down @@