From e4bc2b7d20300f1a04b57b229d887286010de799 Mon Sep 17 00:00:00 2001
From: Jon Edvald <edvald@gmail.com>
Date: Sat, 27 Feb 2021 00:23:03 +0100
Subject: [PATCH] feat(k8s): add garden-build toleration to garden-buildkit
 deployments

---
 .../plugins/kubernetes/container/build/buildkit.ts    |  8 ++++++++
 .../plugins/kubernetes/container/build/buildkit.ts    | 11 ++++++++++-
 docs/guides/in-cluster-building.md                    | 11 +++++++++++
 3 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/core/src/plugins/kubernetes/container/build/buildkit.ts b/core/src/plugins/kubernetes/container/build/buildkit.ts
index 539f9bc5c1..899d556551 100644
--- a/core/src/plugins/kubernetes/container/build/buildkit.ts
+++ b/core/src/plugins/kubernetes/container/build/buildkit.ts
@@ -434,6 +434,14 @@ const baseBuildkitDeployment: KubernetesDeployment = {
             emptyDir: {},
           },
         ],
+        tolerations: [
+          {
+            key: "garden-build",
+            operator: "Equal",
+            value: "true",
+            effect: "NoSchedule",
+          },
+        ],
       },
     },
   },
diff --git a/core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts b/core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts
index 9fb4e64429..929de95307 100644
--- a/core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts
+++ b/core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts
@@ -62,9 +62,18 @@ describe("ensureBuildkit", () => {
       })
 
       // Make sure deployment is there
-      await api.apps.readNamespacedDeployment(buildkitDeploymentName, namespace)
+      const deployment = await api.apps.readNamespacedDeployment(buildkitDeploymentName, namespace)
 
       expect(deployed).to.be.true
+      expect(deployment.spec.template.spec?.tolerations).to.eql([
+        {
+          key: "garden-build",
+          operator: "Equal",
+          value: "true",
+          effect: "NoSchedule",
+          tolerationSeconds: undefined,
+        },
+      ])
     })
 
     it("deploys buildkit with the configured nodeSelector", async () => {
diff --git a/docs/guides/in-cluster-building.md b/docs/guides/in-cluster-building.md
index 2418baa4ef..f678c42084 100644
--- a/docs/guides/in-cluster-building.md
+++ b/docs/guides/in-cluster-building.md
@@ -131,6 +131,17 @@ clusterBuildkit:
 
 You should also set the builder resource requests/limits. For this mode, the resource configuration applies to _each BuildKit deployment_, i.e. for _each project namespace_. See the [builder resources](../reference/providers/kubernetes.md#providersresourcesbuilder) reference for details.
 
+The BuildKit deployments will always have the following toleration set:
+
+```yaml
+key: "garden-build",
+operator: "Equal",
+value: "true",
+effect: "NoSchedule"
+```
+
+This allows you to set corresponding [Taints](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) on cluster nodes to control which nodes builder deployments are deployed to.
+
 ### cluster-docker
 
 The `cluster-docker` mode installs a standalone Docker daemon into your cluster, that is then used for builds across all users of the clusters, along with a handful of other supporting services.