feat(k8s): allow pulling base images when building in cluster

We now use the configured `imagePullSecrets` on the `kubernetes`
provider to authenticate the cluster Docker daemon or Kaniko pods, so
that you can pull base images from private repositories.

This PR includes new integration tests and some helpers that were needed
to facilitate those.

Closes #1236

.circleci/config.yml

@@ -100,14 +100,18 @@ commands:
           name: Install gcloud
           command: |
             mkdir $HOME/gcloud
-            curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz | tar xvz -C $HOME/gcloud
+            curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz | tar xz -C $HOME/gcloud
             $HOME/gcloud/google-cloud-sdk/install.sh --quiet
             echo 'export PATH=$HOME/gcloud/google-cloud-sdk/bin:$PATH' >> $BASH_ENV
       - run:
           name: Configure kubectl context via gcloud and authenticate to Google Container Registry
           command: |
-            echo $GCLOUD_SERVICE_KEY | gcloud auth activate-service-account --key-file=-
-            gcloud --quiet config set project $GOOGLE_PROJECT_ID && gcloud --quiet config set compute/zone $GOOGLE_COMPUTE_ZONE
+            export GOOGLE_APPLICATION_CREDENTIALS=$HOME/gcloud-key.json
+            echo "export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS" >> $BASH_ENV
+            echo $GCLOUD_SERVICE_KEY > $GOOGLE_APPLICATION_CREDENTIALS
+            gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS
+            gcloud --quiet config set project $GOOGLE_PROJECT_ID
+            gcloud --quiet config set compute/zone $GOOGLE_COMPUTE_ZONE
             gcloud --quiet container clusters get-credentials $GOOGLE_CLUSTER_ID --zone $GOOGLE_COMPUTE_ZONE
             gcloud --quiet auth configure-docker
 
@@ -363,6 +367,12 @@ jobs:
       GARDEN_LOGGER_TYPE: basic
     steps:
       - checkout
+      - configure_kubectl_context
+      - run:
+          name: Install system dependencies
+          command: |
+            sudo apt-get update
+            sudo apt install nfs-common
       - run:
           name: Update Node.js
           command: |
@@ -418,6 +428,7 @@ jobs:
           name: Setup remote K8s
           command: |
             gcloud auth activate-service-account --key-file=key.json
+            $env:GOOGLE_APPLICATION_CREDENTIALS = (Get-Location) + '\key.json'
             gcloud --quiet config set project $env:GOOGLE_PROJECT_ID
             gcloud --quiet config set compute/zone $env:GOOGLE_COMPUTE_ZONE
             gcloud --quiet container clusters get-credentials $env:GOOGLE_CLUSTER_ID --zone $env:GOOGLE_COMPUTE_ZONE

bin/encrypt-file.ts

@@ -0,0 +1,57 @@
+#!/usr/bin/env ts-node
+
+/**
+ * Helper script for encrypting files and storing them in the repository. Uses Google Cloud KMS (which devs should
+ * have access to anyway) to encrypt the data, such that it's safe to commit the file to git.
+ *
+ * Usage example: `echo "my data" | ./bin/encrypt-file.ts filename.txt`
+ */
+
+import kms from "@google-cloud/kms"
+import { writeFile } from "fs-extra"
+import { resolve } from "path"
+
+const projectId = "garden-dev-200012"
+const keyRingId = "dev"
+const cryptoKeyId = "dev"
+const locationId = "global"
+
+async function encrypt(filename: string, plaintext: Buffer) {
+  const client = new kms.KeyManagementServiceClient()
+
+  const name = client.cryptoKeyPath(
+    projectId,
+    locationId,
+    keyRingId,
+    cryptoKeyId
+  )
+
+  const [result] = await client.encrypt({ name, plaintext })
+
+  const outputPath = resolve(__dirname, "..", "secrets", filename)
+  await writeFile(outputPath, result.ciphertext)
+
+  console.log(
+    `Encrypted input, result saved to ${outputPath}`
+  )
+}
+
+const args = process.argv.slice(2)
+const filename = args[0]
+
+if (require.main === module) {
+  process.stdin.resume()
+
+  let data = Buffer.from("")
+
+  process.stdin.on("data", (chunk) => {
+    data = Buffer.concat([data, chunk])
+  })
+
+  process.stdin.on("end", function() {
+    encrypt(filename, data).catch((err) => {
+      console.error(err)
+      process.exit(1)
+    })
+  })
+}

docs/guides/in-cluster-building.md

@@ -143,16 +143,30 @@ your own cron jobs.
 
 ## Pulling base images from private registries
 
-Currently, only the _Local Docker_ build mode supports pulling base images from private registries in Dockerfiles. If you see an `ImagePullBackOff` error when using the other build modes, it's likely that it's failing because the Dockerfile for the module contains an entry like this:
+The in-cluster builder may need to be able to pull base images from a private registry, e.g. if your Dockerfile starts something like this:
 
-```console
+```dockerfile
 FROM my-private-registry.com/my-image:tag
 ```
 
-where `my-private-registry` requires authorization.
+where `my-private-registry.com` requires authorization.
 
-This is because Garden currently can't authenticate against the private registry from inside the cluster.
+For this to work, you need to create a registry secret in your cluster (see [this guide](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) for how to create the secret) and then configure the [imagePullSecrets](../reference/providers/kubernetes.md#providersimagepullsecrets) field in your `kubernetes` provider configuration:
 
-We do plan on supporting this for more build modes but it's a non-trivial feature to add. You can track the discussion and progress [in this issue](https://github.com/garden-io/garden/issues/1236).
+```yaml
+kind: Project
+name: my-project
+...
+providers:
+  - name: kubernetes
+    ...
+    imagePullSecrets:
+      # The name of the registry auth secret you created.
+    - name: my-registry-secret
+      # Change this if you store the secret in another namespace.
+      namespace: default
+```
+
+This registry auth secret will then be copied and passed to the in-cluster builder. You can specify as many as you like, and they will be merged together.
 
-Note that you _can reference private images_ in the module config (`image: my-private-registry.com/image:tag`), or in your Helm/Kubernetes modules, as usual. For this you may need to set the `imagePullSecret` directive in the provider configuration.
+> Note: Any time you add or modify imagePullSecrets after first initializing your cluster, you need to run `garden plugins kubernetes cluster-init` again for them to work when pulling base images!