diff --git a/docs/reference/providers/kubernetes.md b/docs/reference/providers/kubernetes.md index 7c84177f6a..d9222489fb 100644 --- a/docs/reference/providers/kubernetes.md +++ b/docs/reference/providers/kubernetes.md @@ -764,6 +764,78 @@ The namespace where the secret is stored. If necessary, the secret may be copied | -------- | -------- | ----------- | | `string` | No | `"default"` | +### `providers[].registryProxyTolerations[]` + +[providers](#providers) > registryProxyTolerations + +For setting tolerations on the registry-proxy when using in-cluster building. +The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node. + +Use this only if you're doing in-cluster building and the nodes in your cluster +have [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/). + +| Type | Required | Default | +| --------------- | -------- | ------- | +| `array[object]` | No | `[]` | + +### `providers[].registryProxyTolerations[].effect` + +[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > effect + +"Effect" indicates the taint effect to match. Empty means match all taint effects. When specified, +allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute". + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `providers[].registryProxyTolerations[].key` + +[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > key + +"Key" is the taint key that the toleration applies to. Empty means match all taint keys. +If the key is empty, operator must be "Exists"; this combination means to match all values and all keys. + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `providers[].registryProxyTolerations[].operator` + +[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > operator + +"Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". Defaults to +"Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a +particular category. + +| Type | Required | Default | +| -------- | -------- | --------- | +| `string` | No | `"Equal"` | + +### `providers[].registryProxyTolerations[].tolerationSeconds` + +[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > tolerationSeconds + +"TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute", +otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate +the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) +by the system. + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `providers[].registryProxyTolerations[].value` + +[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > value + +"Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be empty, +otherwise just a regular string. + +| Type | Required | +| -------- | -------- | +| `string` | No | + ### `providers[].name` [providers](#providers) > name @@ -973,6 +1045,12 @@ providers: secretRef: name: namespace: default + registryProxyTolerations: + - effect: + key: + operator: Equal + tolerationSeconds: + value: name: kubernetes context: deploymentRegistry: diff --git a/docs/reference/providers/local-kubernetes.md b/docs/reference/providers/local-kubernetes.md index 6fe19a883b..da751fd977 100644 --- a/docs/reference/providers/local-kubernetes.md +++ b/docs/reference/providers/local-kubernetes.md @@ -764,6 +764,78 @@ The namespace where the secret is stored. If necessary, the secret may be copied | -------- | -------- | ----------- | | `string` | No | `"default"` | +### `providers[].registryProxyTolerations[]` + +[providers](#providers) > registryProxyTolerations + +For setting tolerations on the registry-proxy when using in-cluster building. +The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node. + +Use this only if you're doing in-cluster building and the nodes in your cluster +have [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/). + +| Type | Required | Default | +| --------------- | -------- | ------- | +| `array[object]` | No | `[]` | + +### `providers[].registryProxyTolerations[].effect` + +[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > effect + +"Effect" indicates the taint effect to match. Empty means match all taint effects. When specified, +allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute". + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `providers[].registryProxyTolerations[].key` + +[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > key + +"Key" is the taint key that the toleration applies to. Empty means match all taint keys. +If the key is empty, operator must be "Exists"; this combination means to match all values and all keys. + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `providers[].registryProxyTolerations[].operator` + +[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > operator + +"Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". Defaults to +"Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a +particular category. + +| Type | Required | Default | +| -------- | -------- | --------- | +| `string` | No | `"Equal"` | + +### `providers[].registryProxyTolerations[].tolerationSeconds` + +[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > tolerationSeconds + +"TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute", +otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate +the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) +by the system. + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `providers[].registryProxyTolerations[].value` + +[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > value + +"Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be empty, +otherwise just a regular string. + +| Type | Required | +| -------- | -------- | +| `string` | No | + ### `providers[].name` [providers](#providers) > name @@ -874,6 +946,12 @@ providers: secretRef: name: namespace: default + registryProxyTolerations: + - effect: + key: + operator: Equal + tolerationSeconds: + value: name: local-kubernetes context: namespace: diff --git a/garden-service/src/plugins/kubernetes/config.ts b/garden-service/src/plugins/kubernetes/config.ts index 4dd5a6b88e..757b6bae44 100644 --- a/garden-service/src/plugins/kubernetes/config.ts +++ b/garden-service/src/plugins/kubernetes/config.ts @@ -54,6 +54,14 @@ interface KubernetesStorage { sync: KubernetesStorageSpec } +export interface Toleration { + effect?: "NoSchedule" | "PreferNoSchedule" | "NoExecute" + key?: string + operator: "Exists" | "Equal" + tolerationSeconds?: number + value?: string +} + export type ContainerBuildMode = "local-docker" | "cluster-docker" | "kaniko" export type DefaultDeploymentStrategy = "rolling" @@ -75,6 +83,7 @@ export interface KubernetesBaseConfig extends ProviderConfig { resources: KubernetesResources storage: KubernetesStorage tlsCertificates: IngressTlsCertificate[] + registryProxyTolerations: Toleration[] _systemServices: string[] } @@ -360,6 +369,46 @@ export const kubernetesConfigBase = providerConfigBaseSchema .description("One or more certificates to use for ingress."), _systemServices: joiArray(joiIdentifier()) .meta({ internal: true }), + registryProxyTolerations: joiArray(joi.object().keys({ + effect: joi.string() + .allow("NoSchedule", "PreferNoSchedule", "NoExecute") + .description(dedent` + "Effect" indicates the taint effect to match. Empty means match all taint effects. When specified, + allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute". + `), + key: joi.string() + .description(dedent` + "Key" is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be "Exists"; this combination means to match all values and all keys. + `), + operator: joi.string() + .allow("Exists", "Equal") + .default("Equal") + .description(dedent` + "Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". Defaults to + "Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a + particular category. + `), + tolerationSeconds: joi.string() + .description(dedent` + "TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute", + otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate + the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) + by the system. + `), + value: joi.string() + .description(dedent` + "Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be empty, + otherwise just a regular string. + `), + })) + .description(dedent` + For setting tolerations on the registry-proxy when using in-cluster building. + The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node. + + Use this only if you're doing in-cluster building and the nodes in your cluster + have [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/). + `), }) export const configSchema = kubernetesConfigBase diff --git a/garden-service/src/plugins/kubernetes/init.ts b/garden-service/src/plugins/kubernetes/init.ts index bef5e27519..408b5f39e1 100644 --- a/garden-service/src/plugins/kubernetes/init.ts +++ b/garden-service/src/plugins/kubernetes/init.ts @@ -279,6 +279,10 @@ export function getKubernetesSystemVariables(config: KubernetesConfig) { "sync-storage-size": megabytesToString(config.storage.sync.size!), "sync-storage-class": syncStorageClass, "sync-volume-name": `garden-sync-${syncStorageClass}`, + + // Stringifying the tolerations since variable values should be primitives. + // Helm handles the decoding automatically. + "registry-proxy-tolerations": JSON.stringify(config.registryProxyTolerations), } } diff --git a/garden-service/src/plugins/kubernetes/kubernetes.ts b/garden-service/src/plugins/kubernetes/kubernetes.ts index 22948983c5..2095e751d9 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes.ts @@ -31,6 +31,7 @@ import { resolve } from "path" import { dedent } from "../../util/string" import { kubernetesModuleSpecSchema } from "./kubernetes-module/config" import { helmModuleSpecSchema, helmModuleOutputsSchema } from "./helm/config" +import { isNumber } from "util" export async function configureProvider( { projectName, projectRoot, config }: ConfigureProviderParams, @@ -77,6 +78,27 @@ export async function configureProvider( config.kubeconfig = resolve(projectRoot, config.kubeconfig) } + for (const { effect, key, operator, tolerationSeconds, value } of config.registryProxyTolerations) { + if (!key && operator !== "Exists") { + throw new ConfigurationError( + `kubernetes: tolerations operator must be 'Exists' if tolerations key is empty`, + { key, operator, config }, + ) + } + if (isNumber(tolerationSeconds) && effect !== "NoExecute") { + throw new ConfigurationError( + `kubernetes: tolerations effect must be 'NoExecute' if toleration seconds is set`, + { tolerationSeconds, effect, config }, + ) + } + if (!!value && operator === "Exists") { + throw new ConfigurationError( + `kubernetes: tolerations value should be empty if tolerations operator is 'Exists'`, + { value, operator, config }, + ) + } + } + return { config } } diff --git a/garden-service/src/plugins/kubernetes/local/config.ts b/garden-service/src/plugins/kubernetes/local/config.ts index 564bc731f6..86e51d1b70 100644 --- a/garden-service/src/plugins/kubernetes/local/config.ts +++ b/garden-service/src/plugins/kubernetes/local/config.ts @@ -161,6 +161,7 @@ export async function configureProvider({ config, log, projectName }: ConfigureP storage: config.storage, setupIngressController: config.setupIngressController, tlsCertificates: config.tlsCertificates, + registryProxyTolerations: config.registryProxyTolerations, _systemServices, } diff --git a/garden-service/static/kubernetes/system/registry-proxy/garden.yml b/garden-service/static/kubernetes/system/registry-proxy/garden.yml index 1a380ddb43..e3b8c0396b 100644 --- a/garden-service/static/kubernetes/system/registry-proxy/garden.yml +++ b/garden-service/static/kubernetes/system/registry-proxy/garden.yml @@ -4,6 +4,7 @@ name: registry-proxy description: DaemonSet that proxies connections to the docker registry service on each node releaseName: garden-registry-proxy values: + tolerations: ${var.registry-proxy-tolerations} registry: hostname: ${var.registry-hostname || "foo"} # tlsSecretName: ${variables.registry-tls-secret-name} diff --git a/garden-service/static/kubernetes/system/registry-proxy/templates/daemonset.yaml b/garden-service/static/kubernetes/system/registry-proxy/templates/daemonset.yaml index 458ce72f43..6a9dffff3d 100644 --- a/garden-service/static/kubernetes/system/registry-proxy/templates/daemonset.yaml +++ b/garden-service/static/kubernetes/system/registry-proxy/templates/daemonset.yaml @@ -47,6 +47,12 @@ spec: - name: envoy-yaml configMap: name: {{ include "registry-proxy.fullname" . }}-envoy + # We expose the tolerations directive to the user. + {{- with .Values.tolerations }} + # Note that we're not using the toYAML function here because the tolerations are JSON stringified (by Garden) + tolerations: + {{- . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -55,7 +61,3 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts index fa60b6348c..32a43ab59c 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts @@ -48,6 +48,7 @@ const basicConfig: KubernetesConfig = { ingressHttpsPort: 443, resources: defaultResources, storage: defaultStorage, + registryProxyTolerations: [], tlsCertificates: [], _systemServices: [], }