diff --git a/core/src/plugins/kubernetes/config.ts b/core/src/plugins/kubernetes/config.ts index ea0f7222a7..9ccbac8db8 100644 --- a/core/src/plugins/kubernetes/config.ts +++ b/core/src/plugins/kubernetes/config.ts @@ -107,6 +107,7 @@ export interface KubernetesConfig extends GenericProviderConfig { buildMode: ContainerBuildMode clusterBuildkit?: { rootless?: boolean + nodeSelector?: StringMap } clusterDocker?: { enableBuildKit?: boolean @@ -329,6 +330,16 @@ export const kubernetesConfigBase = () => Please see [the buildkit docs](https://github.com/moby/buildkit/blob/master/docs/rootless.md) for caveats when using this mode. ` ), + nodeSelector: joiStringMap(joi.string()) + .description( + dedent` + Exposes the \`nodeSelector\` field on the PodSpec of the BuildKit deployment. This allows you to constrain the BuildKit daemon to only run on particular nodes. + + [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes. + ` + ) + .example({ disktype: "ssd" }) + .default(() => ({})), }) .default(() => {}) .description("Configuration options for the `cluster-buildkit` build mode."), @@ -523,9 +534,10 @@ export const kubernetesConfigBase = () => systemNodeSelector: joiStringMap(joi.string()) .description( dedent` - Exposes the \`nodeSelector\` field on the PodSpec of system services. This allows you to constrain - the system services to only run on particular nodes. [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes. - ` + Exposes the \`nodeSelector\` field on the PodSpec of system services. This allows you to constrain the system services to only run on particular nodes. + + [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes. + ` ) .example({ disktype: "ssd" }) .default(() => ({})), diff --git a/core/src/plugins/kubernetes/container/build/buildkit.ts b/core/src/plugins/kubernetes/container/build/buildkit.ts index 5415648c13..cd8a69a3c4 100644 --- a/core/src/plugins/kubernetes/container/build/buildkit.ts +++ b/core/src/plugins/kubernetes/container/build/buildkit.ts @@ -252,7 +252,7 @@ function getDockerBuildFlags(module: ContainerModule) { return args } -function getBuildkitDeployment(provider: KubernetesProvider) { +export function getBuildkitDeployment(provider: KubernetesProvider) { const deployment = cloneDeep(baseBuildkitDeployment) const buildkitContainer = deployment.spec!.template.spec!.containers[0] @@ -289,6 +289,11 @@ function getBuildkitDeployment(provider: KubernetesProvider) { const registryHostname = getRegistryHostname(provider.config) deployment.spec!.template.spec!.containers.push(getSocatContainer(registryHostname)) + // Set the configured nodeSelector, if any + if (provider.config.clusterBuildkit?.nodeSelector) { + deployment.spec!.template.spec!.nodeSelector = provider.config.clusterBuildkit?.nodeSelector + } + return deployment } diff --git a/core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts b/core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts index b204ad9cc7..9fb4e64429 100644 --- a/core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts +++ b/core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts @@ -67,6 +67,28 @@ describe("ensureBuildkit", () => { expect(deployed).to.be.true }) + it("deploys buildkit with the configured nodeSelector", async () => { + try { + await api.apps.deleteNamespacedDeployment(buildkitDeploymentName, namespace) + } catch {} + + const nodeSelector = { "kubernetes.io/os": "linux" } + + provider.config.clusterBuildkit = { nodeSelector } + + await ensureBuildkit({ + ctx, + provider, + log: garden.log, + api, + namespace, + }) + + const deployment = await api.apps.readNamespacedDeployment(buildkitDeploymentName, namespace) + + expect(deployment.spec.template.spec?.nodeSelector).to.eql(nodeSelector) + }) + it("creates a docker auth secret from configured imagePullSecrets", async () => { await ensureBuildkit({ ctx, diff --git a/docs/reference/providers/kubernetes.md b/docs/reference/providers/kubernetes.md index 99587e814c..a50b129b24 100644 --- a/docs/reference/providers/kubernetes.md +++ b/docs/reference/providers/kubernetes.md @@ -50,6 +50,13 @@ providers: # using this mode. rootless: false + # Exposes the `nodeSelector` field on the PodSpec of the BuildKit deployment. This allows you to constrain the + # BuildKit daemon to only run on particular nodes. + # + # [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes + # guide to assigning Pods to nodes. + nodeSelector: {} + # Configuration options for the `cluster-docker` build mode. clusterDocker: # Enable [BuildKit](https://github.com/moby/buildkit) support. This should in most cases work well and be more @@ -250,10 +257,11 @@ providers: # for now). acmeChallengeType: HTTP-01 - # Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain - # the system services to only run on particular nodes. [See - # here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to - # assigning Pods to nodes. + # Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain the system + # services to only run on particular nodes. + # + # [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide + # to assigning Pods to nodes. systemNodeSelector: {} # For setting tolerations on the registry-proxy when using in-cluster building. @@ -422,6 +430,28 @@ Please see [the buildkit docs](https://github.com/moby/buildkit/blob/master/docs | --------- | ------- | -------- | | `boolean` | `false` | No | +### `providers[].clusterBuildkit.nodeSelector` + +[providers](#providers) > [clusterBuildkit](#providersclusterbuildkit) > nodeSelector + +Exposes the `nodeSelector` field on the PodSpec of the BuildKit deployment. This allows you to constrain the BuildKit daemon to only run on particular nodes. + +[See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes. + +| Type | Default | Required | +| -------- | ------- | -------- | +| `object` | `{}` | No | + +Example: + +```yaml +providers: + - clusterBuildkit: + ... + nodeSelector: + disktype: ssd +``` + ### `providers[].clusterDocker` [providers](#providers) > clusterDocker @@ -1288,8 +1318,9 @@ providers: [providers](#providers) > systemNodeSelector -Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain -the system services to only run on particular nodes. [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes. +Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain the system services to only run on particular nodes. + +[See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes. | Type | Default | Required | | -------- | ------- | -------- | diff --git a/docs/reference/providers/local-kubernetes.md b/docs/reference/providers/local-kubernetes.md index 9448503277..efa7958656 100644 --- a/docs/reference/providers/local-kubernetes.md +++ b/docs/reference/providers/local-kubernetes.md @@ -46,6 +46,13 @@ providers: # using this mode. rootless: false + # Exposes the `nodeSelector` field on the PodSpec of the BuildKit deployment. This allows you to constrain the + # BuildKit daemon to only run on particular nodes. + # + # [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes + # guide to assigning Pods to nodes. + nodeSelector: {} + # Configuration options for the `cluster-docker` build mode. clusterDocker: # Enable [BuildKit](https://github.com/moby/buildkit) support. This should in most cases work well and be more @@ -246,10 +253,11 @@ providers: # for now). acmeChallengeType: HTTP-01 - # Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain - # the system services to only run on particular nodes. [See - # here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to - # assigning Pods to nodes. + # Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain the system + # services to only run on particular nodes. + # + # [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide + # to assigning Pods to nodes. systemNodeSelector: {} # For setting tolerations on the registry-proxy when using in-cluster building. @@ -384,6 +392,28 @@ Please see [the buildkit docs](https://github.com/moby/buildkit/blob/master/docs | --------- | ------- | -------- | | `boolean` | `false` | No | +### `providers[].clusterBuildkit.nodeSelector` + +[providers](#providers) > [clusterBuildkit](#providersclusterbuildkit) > nodeSelector + +Exposes the `nodeSelector` field on the PodSpec of the BuildKit deployment. This allows you to constrain the BuildKit daemon to only run on particular nodes. + +[See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes. + +| Type | Default | Required | +| -------- | ------- | -------- | +| `object` | `{}` | No | + +Example: + +```yaml +providers: + - clusterBuildkit: + ... + nodeSelector: + disktype: ssd +``` + ### `providers[].clusterDocker` [providers](#providers) > clusterDocker @@ -1250,8 +1280,9 @@ providers: [providers](#providers) > systemNodeSelector -Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain -the system services to only run on particular nodes. [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes. +Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain the system services to only run on particular nodes. + +[See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes. | Type | Default | Required | | -------- | ------- | -------- |