feat: allow specifying tolerations for registry-proxy

docs/reference/providers/kubernetes.md

@@ -764,6 +764,78 @@ The namespace where the secret is stored. If necessary, the secret may be copied
 | -------- | -------- | ----------- |
 | `string` | No       | `"default"` |
 
+### `providers[].registryProxyTolerations[]`
+
+[providers](#providers) > registryProxyTolerations
+
+For setting tolerations on the registry-proxy when using in-cluster building.
+The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node.
+
+Use this only if you're doing in-cluster building and the nodes in your cluster
+have [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/).
+
+| Type            | Required | Default |
+| --------------- | -------- | ------- |
+| `array[object]` | No       | `[]`    |
+
+### `providers[].registryProxyTolerations[].effect`
+
+[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > effect
+
+"Effect" indicates the taint effect to match. Empty means match all taint effects. When specified,
+allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute".
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
+### `providers[].registryProxyTolerations[].key`
+
+[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > key
+
+"Key" is the taint key that the toleration applies to. Empty means match all taint keys.
+If the key is empty, operator must be "Exists"; this combination means to match all values and all keys.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
+### `providers[].registryProxyTolerations[].operator`
+
+[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > operator
+
+"Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". Defaults to
+"Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a
+particular category.
+
+| Type     | Required | Default   |
+| -------- | -------- | --------- |
+| `string` | No       | `"Equal"` |
+
+### `providers[].registryProxyTolerations[].tolerationSeconds`
+
+[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > tolerationSeconds
+
+"TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute",
+otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate
+the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately)
+by the system.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
+### `providers[].registryProxyTolerations[].value`
+
+[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > value
+
+"Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be empty,
+otherwise just a regular string.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
 ### `providers[].name`
 
 [providers](#providers) > name
@@ -973,6 +1045,12 @@ providers:
         secretRef:
           name:
           namespace: default
+    registryProxyTolerations:
+      - effect:
+        key:
+        operator: Equal
+        tolerationSeconds:
+        value:
     name: kubernetes
     context:
     deploymentRegistry:

docs/reference/providers/local-kubernetes.md

@@ -764,6 +764,78 @@ The namespace where the secret is stored. If necessary, the secret may be copied
 | -------- | -------- | ----------- |
 | `string` | No       | `"default"` |
 
+### `providers[].registryProxyTolerations[]`
+
+[providers](#providers) > registryProxyTolerations
+
+For setting tolerations on the registry-proxy when using in-cluster building.
+The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node.
+
+Use this only if you're doing in-cluster building and the nodes in your cluster
+have [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/).
+
+| Type            | Required | Default |
+| --------------- | -------- | ------- |
+| `array[object]` | No       | `[]`    |
+
+### `providers[].registryProxyTolerations[].effect`
+
+[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > effect
+
+"Effect" indicates the taint effect to match. Empty means match all taint effects. When specified,
+allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute".
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
+### `providers[].registryProxyTolerations[].key`
+
+[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > key
+
+"Key" is the taint key that the toleration applies to. Empty means match all taint keys.
+If the key is empty, operator must be "Exists"; this combination means to match all values and all keys.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
+### `providers[].registryProxyTolerations[].operator`
+
+[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > operator
+
+"Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". Defaults to
+"Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a
+particular category.
+
+| Type     | Required | Default   |
+| -------- | -------- | --------- |
+| `string` | No       | `"Equal"` |
+
+### `providers[].registryProxyTolerations[].tolerationSeconds`
+
+[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > tolerationSeconds
+
+"TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute",
+otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate
+the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately)
+by the system.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
+### `providers[].registryProxyTolerations[].value`
+
+[providers](#providers) > [registryProxyTolerations](#providersregistryproxytolerations) > value
+
+"Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be empty,
+otherwise just a regular string.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
 ### `providers[].name`
 
 [providers](#providers) > name
@@ -874,6 +946,12 @@ providers:
         secretRef:
           name:
           namespace: default
+    registryProxyTolerations:
+      - effect:
+        key:
+        operator: Equal
+        tolerationSeconds:
+        value:
     name: local-kubernetes
     context:
     namespace:

garden-service/src/plugins/kubernetes/config.ts

@@ -54,6 +54,14 @@ interface KubernetesStorage {
   sync: KubernetesStorageSpec
 }
 
+export interface Toleration {
+  effect?: "NoSchedule" | "PreferNoSchedule" | "NoExecute"
+  key?: string
+  operator: "Exists" | "Equal"
+  tolerationSeconds?: number
+  value?: string
+}
+
 export type ContainerBuildMode = "local-docker" | "cluster-docker" | "kaniko"
 
 export type DefaultDeploymentStrategy = "rolling"
@@ -75,6 +83,7 @@ export interface KubernetesBaseConfig extends ProviderConfig {
   resources: KubernetesResources
   storage: KubernetesStorage
   tlsCertificates: IngressTlsCertificate[]
+  registryProxyTolerations: Toleration[]
   _systemServices: string[]
 }
 
@@ -360,6 +369,46 @@ export const kubernetesConfigBase = providerConfigBaseSchema
       .description("One or more certificates to use for ingress."),
     _systemServices: joiArray(joiIdentifier())
       .meta({ internal: true }),
+    registryProxyTolerations: joiArray(joi.object().keys({
+      effect: joi.string()
+        .allow("NoSchedule", "PreferNoSchedule", "NoExecute")
+        .description(dedent`
+          "Effect" indicates the taint effect to match. Empty means match all taint effects. When specified,
+          allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute".
+        `),
+      key: joi.string()
+        .description(dedent`
+          "Key" is the taint key that the toleration applies to. Empty means match all taint keys.
+          If the key is empty, operator must be "Exists"; this combination means to match all values and all keys.
+        `),
+      operator: joi.string()
+        .allow("Exists", "Equal")
+        .default("Equal")
+        .description(dedent`
+          "Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". Defaults to
+          "Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a
+          particular category.
+        `),
+      tolerationSeconds: joi.string()
+        .description(dedent`
+          "TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute",
+          otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate
+          the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately)
+          by the system.
+        `),
+      value: joi.string()
+        .description(dedent`
+          "Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be empty,
+          otherwise just a regular string.
+        `),
+    }))
+      .description(dedent`
+        For setting tolerations on the registry-proxy when using in-cluster building.
+        The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node.
+
+        Use this only if you're doing in-cluster building and the nodes in your cluster
+        have [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/).
+      `),
   })
 
 export const configSchema = kubernetesConfigBase

garden-service/src/plugins/kubernetes/init.ts

@@ -279,6 +279,10 @@ export function getKubernetesSystemVariables(config: KubernetesConfig) {
     "sync-storage-size": megabytesToString(config.storage.sync.size!),
     "sync-storage-class": syncStorageClass,
     "sync-volume-name": `garden-sync-${syncStorageClass}`,
+
+    // Stringifying the tolerations since variable values should be primitives.
+    // Helm handles the decoding automatically.
+    "registry-proxy-tolerations": JSON.stringify(config.registryProxyTolerations),
   }
 }
 

garden-service/src/plugins/kubernetes/kubernetes.ts

@@ -31,6 +31,7 @@ import { resolve } from "path"
 import { dedent } from "../../util/string"
 import { kubernetesModuleSpecSchema } from "./kubernetes-module/config"
 import { helmModuleSpecSchema, helmModuleOutputsSchema } from "./helm/config"
+import { isNumber } from "util"
 
 export async function configureProvider(
   { projectName, projectRoot, config }: ConfigureProviderParams<KubernetesConfig>,
@@ -77,6 +78,27 @@ export async function configureProvider(
     config.kubeconfig = resolve(projectRoot, config.kubeconfig)
   }
 
+  for (const { effect, key, operator, tolerationSeconds, value } of config.registryProxyTolerations) {
+    if (!key && operator !== "Exists") {
+      throw new ConfigurationError(
+        `kubernetes: tolerations operator must be 'Exists' if tolerations key is empty`,
+        { key, operator, config },
+      )
+    }
+    if (isNumber(tolerationSeconds) && effect !== "NoExecute") {
+      throw new ConfigurationError(
+        `kubernetes: tolerations effect must be 'NoExecute' if toleration seconds is set`,
+        { tolerationSeconds, effect, config },
+      )
+    }
+    if (!!value && operator === "Exists") {
+      throw new ConfigurationError(
+        `kubernetes: tolerations value should be empty if tolerations operator is 'Exists'`,
+        { value, operator, config },
+      )
+    }
+  }
+
   return { config }
 }
 

garden-service/src/plugins/kubernetes/local/config.ts

@@ -161,6 +161,7 @@ export async function configureProvider({ config, log, projectName }: ConfigureP
     storage: config.storage,
     setupIngressController: config.setupIngressController,
     tlsCertificates: config.tlsCertificates,
+    registryProxyTolerations: config.registryProxyTolerations,
     _systemServices,
   }
 

garden-service/static/kubernetes/system/registry-proxy/garden.yml

@@ -4,6 +4,7 @@ name: registry-proxy
 description: DaemonSet that proxies connections to the docker registry service on each node
 releaseName: garden-registry-proxy
 values:
+  tolerations: ${var.registry-proxy-tolerations}
   registry:
     hostname: ${var.registry-hostname || "foo"}
     # tlsSecretName: ${variables.registry-tls-secret-name}

garden-service/static/kubernetes/system/registry-proxy/templates/daemonset.yaml

@@ -47,6 +47,12 @@ spec:
         - name: envoy-yaml
           configMap:
             name: {{ include "registry-proxy.fullname" . }}-envoy
+    # We expose the tolerations directive to the user.
+    {{- with .Values.tolerations }}
+      # Note that we're not using the toYAML function here because the tolerations are JSON stringified (by Garden)
+      tolerations:
+        {{- . | nindent 8 }}
+    {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -55,7 +61,3 @@ spec:
       affinity:
         {{- toYaml . | nindent 8 }}
     {{- end }}
-    {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-    {{- end }}

garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts

@@ -48,6 +48,7 @@ const basicConfig: KubernetesConfig = {
   ingressHttpsPort: 443,
   resources: defaultResources,
   storage: defaultStorage,
+  registryProxyTolerations: [],
   tlsCertificates: [],
   _systemServices: [],
 }