diff --git a/docs/reference/providers/kubernetes.md b/docs/reference/providers/kubernetes.md index 521770c783..a8b3c6b7a3 100644 --- a/docs/reference/providers/kubernetes.md +++ b/docs/reference/providers/kubernetes.md @@ -238,6 +238,12 @@ providers: # for now). acmeChallengeType: HTTP-01 + # Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain + # the system services to only run on particular nodes. [See + # here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to + # assigning Pods to nodes. + systemNodeSelector: {} + # For setting tolerations on the registry-proxy when using in-cluster building. # The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node. # @@ -1204,6 +1210,25 @@ providers: acmeChallengeType: "HTTP-01" ``` +### `providers[].systemNodeSelector` + +[providers](#providers) > systemNodeSelector + +Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain +the system services to only run on particular nodes. [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes. + +| Type | Default | Required | +| -------- | ------- | -------- | +| `object` | `{}` | No | + +Example: + +```yaml +providers: + - systemNodeSelector: + disktype: ssd +``` + ### `providers[].registryProxyTolerations[]` [providers](#providers) > registryProxyTolerations diff --git a/docs/reference/providers/local-kubernetes.md b/docs/reference/providers/local-kubernetes.md index 5f948e6bf9..23dc2eeaea 100644 --- a/docs/reference/providers/local-kubernetes.md +++ b/docs/reference/providers/local-kubernetes.md @@ -234,6 +234,12 @@ providers: # for now). acmeChallengeType: HTTP-01 + # Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain + # the system services to only run on particular nodes. [See + # here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to + # assigning Pods to nodes. + systemNodeSelector: {} + # For setting tolerations on the registry-proxy when using in-cluster building. # The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node. # @@ -1172,6 +1178,25 @@ providers: acmeChallengeType: "HTTP-01" ``` +### `providers[].systemNodeSelector` + +[providers](#providers) > systemNodeSelector + +Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain +the system services to only run on particular nodes. [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes. + +| Type | Default | Required | +| -------- | ------- | -------- | +| `object` | `{}` | No | + +Example: + +```yaml +providers: + - systemNodeSelector: + disktype: ssd +``` + ### `providers[].registryProxyTolerations[]` [providers](#providers) > registryProxyTolerations diff --git a/garden-service/src/plugins/kubernetes/config.ts b/garden-service/src/plugins/kubernetes/config.ts index bf4d9b03b0..fec667daf4 100644 --- a/garden-service/src/plugins/kubernetes/config.ts +++ b/garden-service/src/plugins/kubernetes/config.ts @@ -8,7 +8,7 @@ import dedent = require("dedent") -import { joiArray, joiIdentifier, joiProviderName, joi } from "../../config/common" +import { joiArray, joiIdentifier, joiProviderName, joi, joiStringMap } from "../../config/common" import { Provider, providerConfigBaseSchema, ProviderConfig } from "../../config/provider" import { containerRegistryConfigSchema, @@ -104,6 +104,7 @@ export interface KubernetesConfig extends ProviderConfig { kubeconfig?: string namespace?: string registryProxyTolerations: V1Toleration[] + systemNodeSelector: { [key: string]: string } resources: KubernetesResources storage: KubernetesStorage gardenSystemNamespace: string @@ -484,6 +485,15 @@ export const kubernetesConfigBase = providerConfigBaseSchema().keys({ }).description(dedent`cert-manager configuration, for creating and managing TLS certificates. See the [cert-manager guide](https://docs.garden.io/advanced/cert-manager-integration) for details.`), _systemServices: joiArray(joiIdentifier()).meta({ internal: true }), + systemNodeSelector: joiStringMap(joi.string()) + .description( + dedent` + Exposes the \`nodeSelector\` field on the PodSpec of system services. This allows you to constrain + the system services to only run on particular nodes. [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes. + ` + ) + .example({ disktype: "ssd" }) + .default(() => ({})), registryProxyTolerations: joiArray( joi.object().keys({ effect: joi.string().allow("NoSchedule", "PreferNoSchedule", "NoExecute").description(dedent` diff --git a/garden-service/src/plugins/kubernetes/init.ts b/garden-service/src/plugins/kubernetes/init.ts index b8972aff10..a047f39f84 100644 --- a/garden-service/src/plugins/kubernetes/init.ts +++ b/garden-service/src/plugins/kubernetes/init.ts @@ -465,6 +465,7 @@ export function getKubernetesSystemVariables(config: KubernetesConfig) { "registry-proxy-tolerations": registryProxyTolerations, "system-tolerations": systemTolerations, + "system-node-selector": config.systemNodeSelector, } } diff --git a/garden-service/src/plugins/kubernetes/system.ts b/garden-service/src/plugins/kubernetes/system.ts index 25ab53777e..6aa3b22dfb 100644 --- a/garden-service/src/plugins/kubernetes/system.ts +++ b/garden-service/src/plugins/kubernetes/system.ts @@ -55,7 +55,7 @@ export async function getSystemGarden( const conftest: ConftestProviderConfig = { environments: ["default"], name: "conftest-kubernetes", - policyPath: "./policy", + policyPath: "policy", testFailureThreshold: "warn", } diff --git a/garden-service/static/kubernetes/system/build-sync/garden.yml b/garden-service/static/kubernetes/system/build-sync/garden.yml index 17daadf070..39f2756cfe 100644 --- a/garden-service/static/kubernetes/system/build-sync/garden.yml +++ b/garden-service/static/kubernetes/system/build-sync/garden.yml @@ -21,3 +21,4 @@ values: pvc: name: ${var.sync-volume-name} tolerations: ${var.system-tolerations} + nodeSelector: ${var.system-node-selector} diff --git a/garden-service/static/kubernetes/system/docker-daemon/garden.yml b/garden-service/static/kubernetes/system/docker-daemon/garden.yml index 5927e72cac..2bd3c1301d 100644 --- a/garden-service/static/kubernetes/system/docker-daemon/garden.yml +++ b/garden-service/static/kubernetes/system/docker-daemon/garden.yml @@ -23,4 +23,5 @@ values: buildSync: volume: name: ${var.sync-volume-name} - tolerations: ${var.system-tolerations} \ No newline at end of file + tolerations: ${var.system-tolerations} + nodeSelector: ${var.system-node-selector} diff --git a/garden-service/static/kubernetes/system/docker-registry/garden.yml b/garden-service/static/kubernetes/system/docker-registry/garden.yml index 931104d6d8..1c61834c8a 100644 --- a/garden-service/static/kubernetes/system/docker-registry/garden.yml +++ b/garden-service/static/kubernetes/system/docker-registry/garden.yml @@ -27,3 +27,4 @@ values: storageClass: ${var.registry-storage-class} deleteEnabled: true tolerations: ${var.system-tolerations} + nodeSelector: ${var.system-node-selector} diff --git a/garden-service/static/kubernetes/system/ingress-controller/garden.yml b/garden-service/static/kubernetes/system/ingress-controller/garden.yml index ac1ce5db70..614b836d27 100644 --- a/garden-service/static/kubernetes/system/ingress-controller/garden.yml +++ b/garden-service/static/kubernetes/system/ingress-controller/garden.yml @@ -25,5 +25,6 @@ values: omitClusterIP: true minReadySeconds: 1 tolerations: ${var.system-tolerations} + nodeSelector: ${var.system-node-selector} defaultBackend: enabled: false diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/garden.yml b/garden-service/static/kubernetes/system/nfs-provisioner/garden.yml index ada922b4ef..16a695d5d1 100644 --- a/garden-service/static/kubernetes/system/nfs-provisioner/garden.yml +++ b/garden-service/static/kubernetes/system/nfs-provisioner/garden.yml @@ -13,3 +13,4 @@ values: storageClass: name: ${var.sync-storage-class} tolerations: ${var.system-tolerations} + nodeSelector: ${var.system-node-selector} diff --git a/garden-service/test/integ/src/plugins/kubernetes/system.ts b/garden-service/test/integ/src/plugins/kubernetes/system.ts index 231dad6ef8..80e5f50623 100644 --- a/garden-service/test/integ/src/plugins/kubernetes/system.ts +++ b/garden-service/test/integ/src/plugins/kubernetes/system.ts @@ -31,6 +31,24 @@ describe("System services", () => { }) it("should use conftest to check whether system services have a valid config", async () => { + const ctx = garden.getPluginContext(provider) + const variables = getKubernetesSystemVariables(provider.config) + const systemGarden = await getSystemGarden(ctx, variables, garden.log) + const graph = await systemGarden.getConfigGraph(garden.log) + const conftestModuleNames = (await graph.getModules()) + .filter((module) => module.name.startsWith("conftest-")) + .map((m) => m.name) + expect(conftestModuleNames.sort()).to.eql([ + "conftest-build-sync", + "conftest-docker-daemon", + "conftest-docker-registry", + "conftest-ingress-controller", + "conftest-nfs-provisioner", + "conftest-registry-proxy", + ]) + }) + + it("should check whether system modules pass the conftest test", async () => { const ctx = garden.getPluginContext(provider) const variables = getKubernetesSystemVariables(provider.config) const systemGarden = await getSystemGarden(ctx, variables, garden.log) diff --git a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts index 03f08b3071..32d4f85ee0 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts @@ -60,6 +60,7 @@ const basicConfig: KubernetesConfig = { ingressHttpsPort: 443, resources: defaultResources, storage: defaultStorage, + systemNodeSelector: {}, registryProxyTolerations: [], tlsCertificates: [], _systemServices: [], diff --git a/garden-service/test/unit/src/plugins/kubernetes/init.ts b/garden-service/test/unit/src/plugins/kubernetes/init.ts index 6d8c30157f..9913b4dc88 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/init.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/init.ts @@ -53,6 +53,7 @@ const basicConfig: KubernetesConfig = { ingressHttpsPort: 443, resources: defaultResources, storage: defaultStorage, + systemNodeSelector: {}, registryProxyTolerations: [], tlsCertificates: [], _systemServices: [], diff --git a/garden-service/test/unit/src/plugins/kubernetes/kubernetes.ts b/garden-service/test/unit/src/plugins/kubernetes/kubernetes.ts index c1e9304a21..98ceacfa26 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/kubernetes.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/kubernetes.ts @@ -27,6 +27,7 @@ describe("kubernetes configureProvider", () => { ingressHttpsPort: 443, resources: defaultResources, storage: defaultStorage, + systemNodeSelector: {}, registryProxyTolerations: [], tlsCertificates: [], _systemServices: [],