improvement(k8s): get rid of NFS when using kaniko build mode

Closes #1798

core/src/plugins/container/config.ts

@@ -486,7 +486,7 @@ export const containerRegistryConfigSchema = () =>
   }).description(dedent`
     The registry where built containers should be pushed to, and then pulled to the cluster when deploying services.
 
-    Important: If you specify this in combination with \`buildMode: cluster-docker\` or \`buildMode: kaniko\`, you must make sure \`imagePullSecrets\` includes authentication with the specified deployment registry, that has the appropriate write privileges (usually full write access to the configured \`deploymentRegistry.namespace\`).
+    Important: If you specify this in combination with in-cluster building, you must make sure \`imagePullSecrets\` includes authentication with the specified deployment registry, that has the appropriate write privileges (usually full write access to the configured \`deploymentRegistry.namespace\`).
   `)
 
 export interface ContainerService extends GardenService<ContainerModule> {}

core/src/plugins/kubernetes/api.ts

@@ -30,6 +30,7 @@ import {
   Exec,
   Attach,
   V1Deployment,
+  V1Service,
 } from "@kubernetes/client-node"
 import AsyncLock = require("async-lock")
 import request = require("request-promise")
@@ -103,6 +104,14 @@ const apiTypes: { [key: string]: K8sApiConstructor<any> } = {
 }
 
 const crudMap = {
+  Deployment: {
+    cls: new V1Deployment(),
+    group: "apps",
+    read: "readNamespacedDeployment",
+    create: "createNamespacedDeployment",
+    replace: "replaceNamespacedDeployment",
+    delete: "deleteNamespacedDeployment",
+  },
   Secret: {
     cls: new V1Secret(),
     group: "core",
@@ -111,13 +120,13 @@ const crudMap = {
     replace: "replaceNamespacedSecret",
     delete: "deleteNamespacedSecret",
   },
-  Deployment: {
-    cls: new V1Deployment(),
-    group: "apps",
-    read: "readNamespacedDeployment",
-    create: "createNamespacedDeployment",
-    replace: "replaceNamespacedDeployment",
-    delete: "deleteNamespacedDeployment",
+  Service: {
+    cls: new V1Service(),
+    group: "core",
+    read: "readNamespacedService",
+    create: "createNamespacedService",
+    replace: null,
+    delete: "deleteNamespacedService",
   },
 }
 
@@ -547,19 +556,28 @@ export class KubeApi {
 
     log.debug(`Upserting ${kind} ${namespace}/${name}`)
 
-    try {
+    const replace = async () => {
       await api[crudMap[kind].read](name, namespace)
-      await api[crudMap[kind].replace](name, namespace, obj)
+      if (api[crudMap[kind].replace]) {
+        await api[crudMap[kind].replace](name, namespace, obj)
+      } else {
+        // Delete with gracePeriodSeconds = 0
+        await api[crudMap[kind].delete](name, namespace, undefined, undefined, 0)
+        await api[crudMap[kind].create](name, namespace, obj)
+      }
       log.debug(`Replaced ${kind} ${namespace}/${name}`)
+    }
+
+    try {
+      await replace()
     } catch (err) {
       if (err.statusCode === 404) {
         try {
           await api[crudMap[kind].create](namespace, <any>obj)
           log.debug(`Created ${kind} ${namespace}/${name}`)
         } catch (err) {
-          if (err.statusCode === 409) {
-            log.debug(`Patched ${kind} ${namespace}/${name}`)
-            await api[crudMap[kind].replace](name, namespace, obj)
+          if (err.statusCode === 409 || err.statusCode === 422) {
+            await replace()
           } else {
             throw err
           }

core/src/plugins/kubernetes/commands/cluster-init.ts

@@ -48,7 +48,7 @@ export const clusterInit: PluginCommand = {
         ctx: k8sCtx,
         log,
         namespace: systemNamespace,
-        args: ["delete", "--purge", "garden-nfs-provisioner"],
+        args: ["uninstall", "garden-nfs-provisioner"],
       })
     } catch (_) {}
 

core/src/plugins/kubernetes/commands/pull-image.ts

@@ -20,12 +20,17 @@ import { LogEntry } from "../../../logger/log-entry"
 import { containerHelpers } from "../../container/helpers"
 import { RuntimeError } from "../../../exceptions"
 import { PodRunner } from "../run"
-import { dockerAuthSecretKey, dockerAuthSecretName, inClusterRegistryHostname, k8sUtilImageName } from "../constants"
+import {
+  dockerAuthSecretKey,
+  systemDockerAuthSecretName,
+  inClusterRegistryHostname,
+  k8sUtilImageName,
+} from "../constants"
 import { getAppNamespace, getSystemNamespace } from "../namespace"
 import { getRegistryPortForward } from "../container/util"
 import { randomString } from "../../../util/string"
-import { buildkitAuthSecretName, ensureBuilderSecret } from "../container/build/buildkit"
 import { PluginContext } from "../../../plugin-context"
+import { ensureBuilderSecret } from "../container/build/common"
 
 const tmpTarPath = "/tmp/image.tar"
 
@@ -149,18 +154,18 @@ async function pullFromExternalRegistry(
 
   if (buildMode === "cluster-buildkit") {
     namespace = await getAppNamespace(ctx, log, ctx.provider)
-    authSecretName = buildkitAuthSecretName
 
-    await ensureBuilderSecret({
+    const { authSecret } = await ensureBuilderSecret({
       provider: ctx.provider,
       log,
       api,
       namespace,
-      waitForUpdate: false,
     })
+
+    authSecretName = authSecret.metadata.name
   } else {
     namespace = await getSystemNamespace(ctx, ctx.provider, log)
-    authSecretName = dockerAuthSecretName
+    authSecretName = systemDockerAuthSecretName
   }
 
   const imageId = containerHelpers.getDeploymentImageId(module, module.version, ctx.provider.config.deploymentRegistry)

core/src/plugins/kubernetes/commands/uninstall-garden-services.ts

@@ -11,6 +11,8 @@ import { PluginCommand } from "../../../types/plugin/command"
 import { getKubernetesSystemVariables } from "../init"
 import { KubernetesPluginContext } from "../config"
 import { getSystemGarden } from "../system"
+import { getSystemNamespace } from "../namespace"
+import { helm } from "../helm/helm-cli"
 
 export const uninstallGardenServices: PluginCommand = {
   name: "uninstall-garden-services",
@@ -36,10 +38,23 @@ export const uninstallGardenServices: PluginCommand = {
     const serviceNames = services.map((s) => s.name).filter((name) => name !== "nfs-provisioner")
     const serviceStatuses = await actions.deleteServices(log, serviceNames)
 
-    if (k8sCtx.provider.config._systemServices.includes("nfs-provisioner")) {
-      const service = graph.getService("nfs-provisioner")
-      await actions.deleteService({ service, log })
-    }
+    const systemNamespace = await getSystemNamespace(ctx, k8sCtx.provider, log)
+    try {
+      await helm({
+        ctx: k8sCtx,
+        log,
+        namespace: systemNamespace,
+        args: ["uninstall", "garden-nfs-provisioner"],
+      })
+    } catch (_) {}
+    try {
+      await helm({
+        ctx: k8sCtx,
+        log,
+        namespace: systemNamespace,
+        args: ["uninstall", "garden-nfs-provisioner-v2"],
+      })
+    } catch (_) {}
 
     log.info("")
 

core/src/plugins/kubernetes/config.ts

@@ -38,7 +38,7 @@ import { ArtifactSpec } from "../../config/validation"
 import { V1Toleration } from "@kubernetes/client-node"
 import { runPodSpecWhitelist } from "./run"
 
-export const DEFAULT_KANIKO_IMAGE = "gcr.io/kaniko-project/executor:debug-v1.2.0"
+export const DEFAULT_KANIKO_IMAGE = "gcr.io/kaniko-project/executor:v1.6.0-debug"
 export interface ProviderSecretRef {
   name: string
   namespace: string
@@ -116,6 +116,8 @@ export interface KubernetesConfig extends GenericProviderConfig {
   kaniko?: {
     image?: string
     extraFlags?: string[]
+    namespace?: string | null
+    nodeSelector?: StringMap
   }
   context: string
   defaultHostname?: string
@@ -150,7 +152,7 @@ export const defaultResources: KubernetesResources = {
       memory: 8192,
     },
     requests: {
-      cpu: 200,
+      cpu: 100,
       memory: 512,
     },
   },
@@ -362,17 +364,34 @@ export const kubernetesConfigBase = () =>
     kaniko: joi
       .object()
       .keys({
+        extraFlags: joi
+          .array()
+          .items(joi.string())
+          .description(
+            `Specify extra flags to use when building the container image with kaniko. Flags set on \`container\` modules take precedence over these.`
+          ),
         image: joi
           .string()
           .default(DEFAULT_KANIKO_IMAGE)
+          .description(`Change the kaniko image (repository/image:tag) to use when building in kaniko mode.`),
+        namespace: joi
+          .string()
+          .allow(null)
+          .default(defaultSystemNamespace)
           .description(
-            deline`
-            Change the kaniko image (repository/image:tag) to use when building in kaniko mode.
-          `
+            dedent`
+              Choose the namespace where the Kaniko pods will be run. Set to \`null\` to use the project namespace.
+
+              **IMPORTANT: The default namespace will change to the project namespace instead of the garden-system namespace in an upcoming release!**
+            `
           ),
-        extraFlags: joi.array().items(joi.string()).description(deline`
-        Specify extra flags to use when building the container image with kaniko.
-        Flags set on container module take precedence over these.`),
+        nodeSelector: joiStringMap(joi.string()).description(
+          dedent`
+            Exposes the \`nodeSelector\` field on the PodSpec of the Kaniko pods. This allows you to constrain the Kaniko pods to only run on particular nodes.
+
+            [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
+          `
+        ),
       })
       .default(() => {})
       .description("Configuration options for the `kaniko` build mode."),

core/src/plugins/kubernetes/constants.ts

@@ -17,7 +17,7 @@ export const MAX_CONFIGMAP_DATA_SIZE = 1024 * 1024 // max ConfigMap data size is
 // the outputs field, so we cap at 250kB.
 export const MAX_RUN_RESULT_LOG_LENGTH = 250 * 1024
 
-export const dockerAuthSecretName = "builder-docker-config"
+export const systemDockerAuthSecretName = "builder-docker-config"
 export const dockerAuthSecretKey = ".dockerconfigjson"
 export const inClusterRegistryHostname = "127.0.0.1:5000"