From 3748092c51f8288664b32bb103f9b240d9b750cb Mon Sep 17 00:00:00 2001 From: Tomas Ostasevicius <t.ostasevicius@gmail.com> Date: Wed, 11 Aug 2021 22:48:32 +0200 Subject: [PATCH] feat: allow passing additional tolerations to kaniko pods (#2540) * feat: allow passing additional tolerations to kaniko pods * chore: add documentation for the additional kaniko tolerations * chore: add description for kaniko tolerations docs --- core/src/plugins/kubernetes/config.ts | 35 ++++--- .../kubernetes/container/build/kaniko.ts | 6 +- docs/reference/providers/kubernetes.md | 95 +++++++++++++++++++ docs/reference/providers/local-kubernetes.md | 95 +++++++++++++++++++ 4 files changed, 214 insertions(+), 17 deletions(-) diff --git a/core/src/plugins/kubernetes/config.ts b/core/src/plugins/kubernetes/config.ts index 9e70c47a34..f46faaaf19 100644 --- a/core/src/plugins/kubernetes/config.ts +++ b/core/src/plugins/kubernetes/config.ts @@ -120,6 +120,7 @@ export interface KubernetesConfig extends BaseProviderConfig { extraFlags?: string[] namespace?: string | null nodeSelector?: StringMap + tolerations?: V1Toleration[] } context: string defaultHostname?: string @@ -420,6 +421,9 @@ export const kubernetesConfigBase = () => [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes. ` ), + tolerations: joiSparseArray(tolerationSchema()).description( + "Specify tolerations to apply to each Kaniko Pod. Useful to control which nodes in a cluster can run builds." + ), }) .default(() => {}) .description("Configuration options for the `kaniko` build mode."), @@ -615,39 +619,40 @@ export const kubernetesConfigBase = () => ) .example({ disktype: "ssd" }) .default(() => ({})), - registryProxyTolerations: joiSparseArray( - joi.object().keys({ - effect: joi.string().allow("NoSchedule", "PreferNoSchedule", "NoExecute").description(dedent` + registryProxyTolerations: joiSparseArray(tolerationSchema()).description(dedent` + For setting tolerations on the registry-proxy when using in-cluster building. + The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node. + + Use this only if you're doing in-cluster building and the nodes in your cluster + have [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/). + `), + }) + +export const tolerationSchema = () => + joi.object().keys({ + effect: joi.string().allow("NoSchedule", "PreferNoSchedule", "NoExecute").description(dedent` "Effect" indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute". `), - key: joi.string().description(dedent` + key: joi.string().description(dedent` "Key" is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be "Exists"; this combination means to match all values and all keys. `), - operator: joi.string().allow("Exists", "Equal").default("Equal").description(dedent` + operator: joi.string().allow("Exists", "Equal").default("Equal").description(dedent` "Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". Defaults to "Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. `), - tolerationSeconds: joi.string().description(dedent` + tolerationSeconds: joi.string().description(dedent` "TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute", otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. `), - value: joi.string().description(dedent` + value: joi.string().description(dedent` "Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be empty, otherwise just a regular string. `), - }) - ).description(dedent` - For setting tolerations on the registry-proxy when using in-cluster building. - The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node. - - Use this only if you're doing in-cluster building and the nodes in your cluster - have [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/). - `), }) export const namespaceSchema = () => diff --git a/core/src/plugins/kubernetes/container/build/kaniko.ts b/core/src/plugins/kubernetes/container/build/kaniko.ts index 7c703802cd..9af3628cad 100644 --- a/core/src/plugins/kubernetes/container/build/kaniko.ts +++ b/core/src/plugins/kubernetes/container/build/kaniko.ts @@ -347,6 +347,7 @@ async function runKaniko({ } const kanikoImage = provider.config.kaniko?.image || DEFAULT_KANIKO_IMAGE + const kanikoTolerations = [...(provider.config.kaniko?.tolerations || []), builderToleration] const utilHostname = `${utilDeploymentName}.${utilNamespace}.svc.cluster.local` const sourceUrl = `rsync://${utilHostname}:${utilRsyncPort}/volume/${ctx.workingCopyId}/${module.name}/` @@ -433,7 +434,7 @@ async function runKaniko({ }, }, ], - tolerations: [builderToleration], + tolerations: kanikoTolerations, } if (provider.config.deploymentRegistry?.hostname === inClusterRegistryHostname) { @@ -516,6 +517,7 @@ async function runKaniko({ } export function getUtilManifests(provider: KubernetesProvider, authSecretName: string) { + const kanikoTolerations = [...(provider.config.kaniko?.tolerations || []), builderToleration] const deployment: KubernetesDeployment = { apiVersion: "apps/v1", kind: "Deployment", @@ -558,7 +560,7 @@ export function getUtilManifests(provider: KubernetesProvider, authSecretName: s emptyDir: {}, }, ], - tolerations: [builderToleration], + tolerations: kanikoTolerations, }, }, }, diff --git a/docs/reference/providers/kubernetes.md b/docs/reference/providers/kubernetes.md index c79c6acc6d..fd70838e84 100644 --- a/docs/reference/providers/kubernetes.md +++ b/docs/reference/providers/kubernetes.md @@ -81,6 +81,33 @@ providers: # guide to assigning Pods to nodes. nodeSelector: + # Specify tolerations to apply to each Kaniko Pod. Useful to control which nodes in a cluster can run builds. + tolerations: + - # "Effect" indicates the taint effect to match. Empty means match all taint effects. When specified, + # allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute". + effect: + + # "Key" is the taint key that the toleration applies to. Empty means match all taint keys. + # If the key is empty, operator must be "Exists"; this combination means to match all values and all keys. + key: + + # "Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". + # Defaults to + # "Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a + # particular category. + operator: Equal + + # "TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute", + # otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate + # the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) + # by the system. + tolerationSeconds: + + # "Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be + # empty, + # otherwise just a regular string. + value: + # A default hostname to use when no hostname is explicitly configured for a service. defaultHostname: @@ -507,6 +534,74 @@ Exposes the `nodeSelector` field on the PodSpec of the Kaniko pods. This allows | -------- | -------- | | `object` | No | +### `providers[].kaniko.tolerations[]` + +[providers](#providers) > [kaniko](#providerskaniko) > tolerations + +Specify tolerations to apply to each Kaniko Pod. Useful to control which nodes in a cluster can run builds. + +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | + +### `providers[].kaniko.tolerations[].effect` + +[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > effect + +"Effect" indicates the taint effect to match. Empty means match all taint effects. When specified, +allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute". + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `providers[].kaniko.tolerations[].key` + +[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > key + +"Key" is the taint key that the toleration applies to. Empty means match all taint keys. +If the key is empty, operator must be "Exists"; this combination means to match all values and all keys. + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `providers[].kaniko.tolerations[].operator` + +[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > operator + +"Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". Defaults to +"Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a +particular category. + +| Type | Default | Required | +| -------- | --------- | -------- | +| `string` | `"Equal"` | No | + +### `providers[].kaniko.tolerations[].tolerationSeconds` + +[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > tolerationSeconds + +"TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute", +otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate +the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) +by the system. + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `providers[].kaniko.tolerations[].value` + +[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > value + +"Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be empty, +otherwise just a regular string. + +| Type | Required | +| -------- | -------- | +| `string` | No | + ### `providers[].defaultHostname` [providers](#providers) > defaultHostname diff --git a/docs/reference/providers/local-kubernetes.md b/docs/reference/providers/local-kubernetes.md index b5ed5d1d01..4fabd073f2 100644 --- a/docs/reference/providers/local-kubernetes.md +++ b/docs/reference/providers/local-kubernetes.md @@ -77,6 +77,33 @@ providers: # guide to assigning Pods to nodes. nodeSelector: + # Specify tolerations to apply to each Kaniko Pod. Useful to control which nodes in a cluster can run builds. + tolerations: + - # "Effect" indicates the taint effect to match. Empty means match all taint effects. When specified, + # allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute". + effect: + + # "Key" is the taint key that the toleration applies to. Empty means match all taint keys. + # If the key is empty, operator must be "Exists"; this combination means to match all values and all keys. + key: + + # "Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". + # Defaults to + # "Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a + # particular category. + operator: Equal + + # "TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute", + # otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate + # the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) + # by the system. + tolerationSeconds: + + # "Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be + # empty, + # otherwise just a regular string. + value: + # A default hostname to use when no hostname is explicitly configured for a service. defaultHostname: @@ -469,6 +496,74 @@ Exposes the `nodeSelector` field on the PodSpec of the Kaniko pods. This allows | -------- | -------- | | `object` | No | +### `providers[].kaniko.tolerations[]` + +[providers](#providers) > [kaniko](#providerskaniko) > tolerations + +Specify tolerations to apply to each Kaniko Pod. Useful to control which nodes in a cluster can run builds. + +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | + +### `providers[].kaniko.tolerations[].effect` + +[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > effect + +"Effect" indicates the taint effect to match. Empty means match all taint effects. When specified, +allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute". + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `providers[].kaniko.tolerations[].key` + +[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > key + +"Key" is the taint key that the toleration applies to. Empty means match all taint keys. +If the key is empty, operator must be "Exists"; this combination means to match all values and all keys. + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `providers[].kaniko.tolerations[].operator` + +[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > operator + +"Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". Defaults to +"Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a +particular category. + +| Type | Default | Required | +| -------- | --------- | -------- | +| `string` | `"Equal"` | No | + +### `providers[].kaniko.tolerations[].tolerationSeconds` + +[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > tolerationSeconds + +"TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute", +otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate +the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) +by the system. + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `providers[].kaniko.tolerations[].value` + +[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > value + +"Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be empty, +otherwise just a regular string. + +| Type | Required | +| -------- | -------- | +| `string` | No | + ### `providers[].defaultHostname` [providers](#providers) > defaultHostname