From 92bdeb0e093581baa6145cbd15f6d59659240ac0 Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Fri, 29 Nov 2019 23:20:15 +0100 Subject: [PATCH] feat(k8s): allow pulling base images when building in cluster We now use the configured `imagePullSecrets` on the `kubernetes` provider to authenticate the cluster Docker daemon or Kaniko pods, so that you can pull base images from private repositories. This PR includes new integration tests and some helpers that were needed to facilitate those. Closes #1236 --- .circleci/config.yml | 17 +- bin/encrypt-file.ts | 57 +++ docs/guides/in-cluster-building.md | 26 +- garden-service/package-lock.json | 452 ++++++++++++++++++ garden-service/package.json | 2 + garden-service/src/plugins/kubernetes/api.ts | 24 +- .../src/plugins/kubernetes/config.ts | 7 +- .../src/plugins/kubernetes/constants.ts | 3 + .../src/plugins/kubernetes/container/build.ts | 28 +- .../kubernetes/container/deployment.ts | 2 +- .../plugins/kubernetes/container/ingress.ts | 6 +- garden-service/src/plugins/kubernetes/init.ts | 119 ++++- .../src/plugins/kubernetes/kubernetes.ts | 6 +- .../src/plugins/kubernetes/local/config.ts | 93 ++-- .../src/plugins/kubernetes/secrets.ts | 21 +- .../docker-daemon/templates/deployment.yaml | 17 +- .../data/test-projects/container/garden.yml | 33 +- .../container/inaccessible-base/Dockerfile | 3 + .../container/inaccessible-base/garden.yml | 4 + .../container/private-base/Dockerfile | 3 + .../container/private-base/foo.txt | 0 .../container/private-base/garden.yml | 4 + .../container/simple-service/Dockerfile | 4 +- .../test-projects/container/simple/garden.yml | 2 +- garden-service/test/helpers.ts | 13 +- garden-service/test/integ/helpers.ts | 20 + .../src/plugins/kubernetes/container/build.ts | 207 ++++++++ .../test/integ/src/plugins/kubernetes/util.ts | 4 +- garden-service/test/mocha.integ.opts | 2 +- .../unit/src/plugins/container/container.ts | 2 +- .../unit/src/plugins/container/helpers.ts | 2 +- garden-service/test/unit/src/plugins/exec.ts | 2 +- .../plugins/kubernetes/container/ingress.ts | 35 +- .../plugins/kubernetes/container/service.ts | 2 +- package-lock.json | 443 +++++++++++++++++ package.json | 2 + secrets/README.md | 5 + secrets/test-docker-auth.json | Bin 0 -> 586 bytes 38 files changed, 1516 insertions(+), 156 deletions(-) create mode 100755 bin/encrypt-file.ts create mode 100644 garden-service/test/data/test-projects/container/inaccessible-base/Dockerfile create mode 100644 garden-service/test/data/test-projects/container/inaccessible-base/garden.yml create mode 100644 garden-service/test/data/test-projects/container/private-base/Dockerfile create mode 100644 garden-service/test/data/test-projects/container/private-base/foo.txt create mode 100644 garden-service/test/data/test-projects/container/private-base/garden.yml create mode 100644 garden-service/test/integ/helpers.ts create mode 100644 garden-service/test/integ/src/plugins/kubernetes/container/build.ts create mode 100644 secrets/README.md create mode 100644 secrets/test-docker-auth.json diff --git a/.circleci/config.yml b/.circleci/config.yml index 2ad2c47af2..0cecbfde2d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,14 +100,18 @@ commands: name: Install gcloud command: | mkdir $HOME/gcloud - curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz | tar xvz -C $HOME/gcloud + curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz | tar xz -C $HOME/gcloud $HOME/gcloud/google-cloud-sdk/install.sh --quiet echo 'export PATH=$HOME/gcloud/google-cloud-sdk/bin:$PATH' >> $BASH_ENV - run: name: Configure kubectl context via gcloud and authenticate to Google Container Registry command: | - echo $GCLOUD_SERVICE_KEY | gcloud auth activate-service-account --key-file=- - gcloud --quiet config set project $GOOGLE_PROJECT_ID && gcloud --quiet config set compute/zone $GOOGLE_COMPUTE_ZONE + export GOOGLE_APPLICATION_CREDENTIALS=$HOME/gcloud-key.json + echo "export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS" >> $BASH_ENV + echo $GCLOUD_SERVICE_KEY > $GOOGLE_APPLICATION_CREDENTIALS + gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS + gcloud --quiet config set project $GOOGLE_PROJECT_ID + gcloud --quiet config set compute/zone $GOOGLE_COMPUTE_ZONE gcloud --quiet container clusters get-credentials $GOOGLE_CLUSTER_ID --zone $GOOGLE_COMPUTE_ZONE gcloud --quiet auth configure-docker @@ -334,6 +338,12 @@ jobs: GARDEN_LOGGER_TYPE: basic steps: - checkout + - configure_kubectl_context + - run: + name: Install system dependencies + command: | + sudo apt-get update + sudo apt install nfs-common - run: name: Update Node.js command: | @@ -389,6 +399,7 @@ jobs: name: Setup remote K8s command: | gcloud auth activate-service-account --key-file=key.json + $env:GOOGLE_APPLICATION_CREDENTIALS = (Get-Location) + '\key.json' gcloud --quiet config set project $env:GOOGLE_PROJECT_ID gcloud --quiet config set compute/zone $env:GOOGLE_COMPUTE_ZONE gcloud --quiet container clusters get-credentials $env:GOOGLE_CLUSTER_ID --zone $env:GOOGLE_COMPUTE_ZONE diff --git a/bin/encrypt-file.ts b/bin/encrypt-file.ts new file mode 100755 index 0000000000..aa85ef9a13 --- /dev/null +++ b/bin/encrypt-file.ts @@ -0,0 +1,57 @@ +#!/usr/bin/env ts-node + +/** + * Helper script for encrypting files and storing them in the repository. Uses Google Cloud KMS (which devs should + * have access to anyway) to encrypt the data, such that it's safe to commit the file to git. + * + * Usage example: `echo "my data" | ./bin/encrypt-file.ts filename.txt` + */ + +import kms from "@google-cloud/kms" +import { writeFile } from "fs-extra" +import { resolve } from "path" + +const projectId = "garden-dev-200012" +const keyRingId = "dev" +const cryptoKeyId = "dev" +const locationId = "global" + +async function encrypt(filename: string, plaintext: Buffer) { + const client = new kms.KeyManagementServiceClient() + + const name = client.cryptoKeyPath( + projectId, + locationId, + keyRingId, + cryptoKeyId + ) + + const [result] = await client.encrypt({ name, plaintext }) + + const outputPath = resolve(__dirname, "..", "secrets", filename) + await writeFile(outputPath, result.ciphertext) + + console.log( + `Encrypted input, result saved to ${outputPath}` + ) +} + +const args = process.argv.slice(2) +const filename = args[0] + +if (require.main === module) { + process.stdin.resume() + + let data = Buffer.from("") + + process.stdin.on("data", (chunk) => { + data = Buffer.concat([data, chunk]) + }) + + process.stdin.on("end", function() { + encrypt(filename, data).catch((err) => { + console.error(err) + process.exit(1) + }) + }) +} diff --git a/docs/guides/in-cluster-building.md b/docs/guides/in-cluster-building.md index 19458c8edd..3920186a0e 100644 --- a/docs/guides/in-cluster-building.md +++ b/docs/guides/in-cluster-building.md @@ -143,16 +143,30 @@ your own cron jobs. ## Pulling base images from private registries -Currently, only the _Local Docker_ build mode supports pulling base images from private registries in Dockerfiles. If you see an `ImagePullBackOff` error when using the other build modes, it's likely that it's failing because the Dockerfile for the module contains an entry like this: +The in-cluster builder may need to be able to pull base images from a private registry, e.g. if your Dockerfile starts something like this: -```console +```dockerfile FROM my-private-registry.com/my-image:tag ``` -where `my-private-registry` requires authorization. +where `my-private-registry.com` requires authorization. -This is because Garden currently can't authenticate against the private registry from inside the cluster. +For this to work, you need to create a registry secret in your cluster (see [this guide](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) for how to create the secret) and then configure the [imagePullSecrets](../reference/providers/kubernetes.md#providersimagepullsecrets) field in your `kubernetes` provider configuration: -We do plan on supporting this for more build modes but it's a non-trivial feature to add. You can track the discussion and progress [in this issue](https://github.com/garden-io/garden/issues/1236). +```yaml +kind: Project +name: my-project +... +providers: + - name: kubernetes + ... + imagePullSecrets: + # The name of the registry auth secret you created. + - name: my-registry-secret + # Change this if you store the secret in another namespace. + namespace: default +``` + +This registry auth secret will then be copied and passed to the in-cluster builder. You can specify as many as you like, and they will be merged together. -Note that you _can reference private images_ in the module config (`image: my-private-registry.com/image:tag`), or in your Helm/Kubernetes modules, as usual. For this you may need to set the `imagePullSecret` directive in the provider configuration. +> Note: Any time you add or modify imagePullSecrets after first initializing your cluster, you need to run `garden plugins kubernetes cluster-init` again for them to work when pulling base images! diff --git a/garden-service/package-lock.json b/garden-service/package-lock.json index 7183622b90..dbf42a8d52 100644 --- a/garden-service/package-lock.json +++ b/garden-service/package-lock.json @@ -490,6 +490,42 @@ } } }, + "@google-cloud/kms": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@google-cloud/kms/-/kms-1.5.3.tgz", + "integrity": "sha512-egk9hyHLj1w0i7gSXWsUQBZ68xEBk5LSO3KkKgKixoZh9OFboE4Nmh7UY+HBMirIqKznopSfEVOP+Az/ONY4lw==", + "dev": true, + "requires": { + "google-gax": "^1.7.5" + } + }, + "@grpc/grpc-js": { + "version": "0.6.9", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-0.6.9.tgz", + "integrity": "sha512-r1nDOEEiYmAsVYBaS4DPPqdwPOXPw7YhVOnnpPdWhlNtKbYzPash6DqWTTza9gBiYMA5d2Wiq6HzrPqsRaP4yA==", + "dev": true, + "requires": { + "semver": "^6.2.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, + "@grpc/proto-loader": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.5.3.tgz", + "integrity": "sha512-8qvUtGg77G2ZT2HqdqYoM/OY97gQd/0crSG34xNmZ4ZOsv3aQT/FQV9QfZPazTGna6MIoyUd+u6AxsoZjJ/VMQ==", + "dev": true, + "requires": { + "lodash.camelcase": "^4.3.0", + "protobufjs": "^6.8.6" + } + }, "@hapi/address": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz", @@ -605,6 +641,70 @@ "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz", "integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==" }, + "@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=", + "dev": true + }, + "@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "dev": true + }, + "@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "dev": true + }, + "@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=", + "dev": true + }, + "@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "dev": true, + "requires": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=", + "dev": true + }, + "@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=", + "dev": true + }, + "@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=", + "dev": true + }, + "@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=", + "dev": true + }, + "@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=", + "dev": true + }, "@segment/loosely-validate-event": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@segment/loosely-validate-event/-/loosely-validate-event-2.0.0.tgz", @@ -786,6 +886,22 @@ "@types/node": "*" } }, + "@types/google-cloud__kms": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@types/google-cloud__kms/-/google-cloud__kms-1.5.0.tgz", + "integrity": "sha512-IFQUDHKViQ9RKhmGQFmslpKt6vJzSocZ8u5U4SA6YeQ8fGvoKJ3gkULz63oR+pY5bEvoCReUb8w0blSt47m5IA==", + "dev": true, + "requires": { + "@types/google-protobuf": "*", + "@types/node": "*" + } + }, + "@types/google-protobuf": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/@types/google-protobuf/-/google-protobuf-3.7.2.tgz", + "integrity": "sha512-ifFemzjNchFBCtHS6bZNhSZCBu7tbtOe0e8qY0z2J4HtFXmPJjm6fXSaQsTG7yhShBEZtt2oP/bkwu5k+emlkQ==", + "dev": true + }, "@types/hapi__joi": { "version": "15.0.4", "resolved": "https://registry.npmjs.org/@types/hapi__joi/-/hapi__joi-15.0.4.tgz", @@ -941,6 +1057,12 @@ "integrity": "sha512-ijGqzZt/b7BfzcK9vTrS6MFljQRPn5BFWOx8oE0GYxribu6uV+aA9zZuXI1zc/etK9E8nrgdoF2+LgUw7+9tJQ==", "dev": true }, + "@types/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz", + "integrity": "sha512-1w52Nyx4Gq47uuu0EVcsHBxZFJgurQ+rTKS3qMHxR1GY2T8c2AJYd6vZoZ9q1rupaDjU0yT+Jc2XTyXkjeMA+Q==", + "dev": true + }, "@types/mime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.1.tgz", @@ -1224,6 +1346,15 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, + "abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, + "requires": { + "event-target-shim": "^5.0.0" + } + }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -1248,6 +1379,15 @@ } } }, + "agent-base": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", + "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", + "dev": true, + "requires": { + "es6-promisify": "^5.0.0" + } + }, "aggregate-error": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-1.0.0.tgz", @@ -1925,6 +2065,12 @@ "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz", "integrity": "sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==" }, + "bignumber.js": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", + "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==", + "dev": true + }, "binary": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", @@ -2012,6 +2158,12 @@ "integrity": "sha1-WWFrSYME1Var1GaWayLu2j7KX74=", "dev": true }, + "buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=", + "dev": true + }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -3890,6 +4042,15 @@ "safer-buffer": "^2.1.0" } }, + "ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dev": true, + "requires": { + "safe-buffer": "^5.0.1" + } + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -4030,6 +4191,15 @@ "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==" }, + "es6-promisify": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", + "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", + "dev": true, + "requires": { + "es6-promise": "^4.0.3" + } + }, "es6-symbol": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.1.tgz", @@ -4110,6 +4280,12 @@ "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", "dev": true }, + "event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dev": true + }, "eventemitter2": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-5.0.1.tgz", @@ -4395,6 +4571,12 @@ "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.6.tgz", "integrity": "sha512-q8BZ89jjc+mz08rSxROs8VsrBBcn1SIw1kq9NjolL509tkABRk9io01RAjSaEv1Xb2uFLt8VtRiZbGp5H8iDtg==" }, + "fast-text-encoding": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz", + "integrity": "sha512-R9bHCvweUxxwkDwhjav5vxpFvdPGlVngtqmx4pIZfSUhM/Q4NiIUHB456BAf+Q1Nwu3HEZYONtu+Rya+af4jiQ==", + "dev": true + }, "fecha": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz", @@ -4665,6 +4847,37 @@ "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", "dev": true }, + "gaxios": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-2.1.0.tgz", + "integrity": "sha512-Gtpb5sdQmb82sgVkT2GnS2n+Kx4dlFwbeMYcDlD395aEvsLCSQXJJcHt7oJ2LrGxDEAeiOkK79Zv2A8Pzt6CFg==", + "dev": true, + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^3.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.3.0" + }, + "dependencies": { + "is-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", + "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", + "dev": true + } + } + }, + "gcp-metadata": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-3.2.2.tgz", + "integrity": "sha512-vR7kcJMCYJG/mYWp/a1OszdOqnLB/XW1GorWW1hc1lWVNL26L497zypWb9cG0CYDQ4Bl1Wk0+fSZFFjwJlTQgQ==", + "dev": true, + "requires": { + "gaxios": "^2.1.0", + "json-bigint": "^0.3.0" + } + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -5627,6 +5840,84 @@ "sparkles": "^1.0.0" } }, + "google-auth-library": { + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-5.5.1.tgz", + "integrity": "sha512-zCtjQccWS/EHYyFdXRbfeSGM/gW+d7uMAcVnvXRnjBXON5ijo6s0nsObP0ifqileIDSbZjTlLtgo+UoN8IFJcg==", + "dev": true, + "requires": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "fast-text-encoding": "^1.0.0", + "gaxios": "^2.1.0", + "gcp-metadata": "^3.2.0", + "gtoken": "^4.1.0", + "jws": "^3.1.5", + "lru-cache": "^5.0.0" + }, + "dependencies": { + "arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "dev": true + }, + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "requires": { + "yallist": "^3.0.2" + } + }, + "yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true + } + } + }, + "google-gax": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-1.11.1.tgz", + "integrity": "sha512-v/APF2G5h2nS5R/1DW2vsgloaMu2/B3xjHdAptR1yUwZpEd9rxPTlhqosrjl/VRu+gWGr9JZN19ZgJTXQ/Db6Q==", + "dev": true, + "requires": { + "@grpc/grpc-js": "0.6.9", + "@grpc/proto-loader": "^0.5.1", + "@types/long": "^4.0.0", + "abort-controller": "^3.0.0", + "duplexify": "^3.6.0", + "google-auth-library": "^5.0.0", + "is-stream-ended": "^0.1.4", + "lodash.at": "^4.6.0", + "lodash.has": "^4.5.2", + "node-fetch": "^2.6.0", + "protobufjs": "^6.8.8", + "retry-request": "^4.0.0", + "semver": "^6.0.0", + "walkdir": "^0.4.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, + "google-p12-pem": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-2.0.3.tgz", + "integrity": "sha512-Tq2kBCANxYYPxaBpTgCpRfdoPs9+/lNzc/Iaee4kuMVW5ascD+HwhpBsTLwH85C9Ev4qfB8KKHmpPQYyD2vg2w==", + "dev": true, + "requires": { + "node-forge": "^0.9.0" + } + }, "graceful-fs": { "version": "4.1.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", @@ -5648,6 +5939,26 @@ "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==" }, + "gtoken": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-4.1.3.tgz", + "integrity": "sha512-ofW+FiXjswyKdkjMcDbe6E4K7cDDdE82dGDhZIc++kUECqaE7MSErf6arJPAjcnYn1qxE1/Ti06qQuqgVusovQ==", + "dev": true, + "requires": { + "gaxios": "^2.1.0", + "google-p12-pem": "^2.0.0", + "jws": "^3.1.5", + "mime": "^2.2.0" + }, + "dependencies": { + "mime": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", + "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==", + "dev": true + } + } + }, "gulp": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/gulp/-/gulp-4.0.2.tgz", @@ -6176,6 +6487,16 @@ "sshpk": "^1.7.0" } }, + "https-proxy-agent": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-3.0.1.tgz", + "integrity": "sha512-+ML2Rbh6DAuee7d07tYGEKOEi2voWPUGan+ExdPbPW6Z3svq+JCqr0v8WmKPOkz1vOVykPCBSuobe7G8GJUtVg==", + "dev": true, + "requires": { + "agent-base": "^4.3.0", + "debug": "^3.1.0" + } + }, "human-signals": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz", @@ -6567,6 +6888,12 @@ "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" }, + "is-stream-ended": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-stream-ended/-/is-stream-ended-0.1.4.tgz", + "integrity": "sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==", + "dev": true + }, "is-subset": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/is-subset/-/is-subset-0.1.1.tgz", @@ -6806,6 +7133,15 @@ "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", "dev": true }, + "json-bigint": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz", + "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=", + "dev": true, + "requires": { + "bignumber.js": "^7.0.0" + } + }, "json-buffer": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", @@ -6904,6 +7240,27 @@ "integrity": "sha1-h/zPrv/AtozRnVX2cilD+SnqNeo=", "dev": true }, + "jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "dev": true, + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "dev": true, + "requires": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } + }, "keygrip": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", @@ -7271,6 +7628,18 @@ "integrity": "sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0=", "dev": true }, + "lodash.at": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.at/-/lodash.at-4.6.0.tgz", + "integrity": "sha1-k83OZk8KGZTqM9181A4jr9EbD/g=", + "dev": true + }, + "lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=", + "dev": true + }, "lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -7292,6 +7661,12 @@ "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", "dev": true }, + "lodash.has": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz", + "integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI=", + "dev": true + }, "lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", @@ -8176,6 +8551,12 @@ } } }, + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==", + "dev": true + }, "node-forge": { "version": "0.9.1", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz", @@ -9269,6 +9650,35 @@ } } }, + "protobufjs": { + "version": "6.8.8", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", + "integrity": "sha512-AAmHtD5pXgZfi7GMpllpO3q1Xw1OYldr+dMUlAnffGTAhqkg72WdmSY71uKBF/JuyiKs8psYbtKrhi0ASCD8qw==", + "dev": true, + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.0", + "@types/node": "^10.1.0", + "long": "^4.0.0" + }, + "dependencies": { + "@types/node": { + "version": "10.17.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.6.tgz", + "integrity": "sha512-0a2X6cgN3RdPBL2MIlR6Lt0KlM7fOFsutuXcdglcOq6WvLnYXgPQSh0Mx6tO1KCAE8MxbHSOSTWDoUxRq+l3DA==", + "dev": true + } + } + }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -9845,6 +10255,42 @@ "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=" }, + "retry-request": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz", + "integrity": "sha512-BINDzVtLI2BDukjWmjAIRZ0oglnCAkpP2vQjM3jdLhmT62h0xnQgciPwBRDAvHqpkPT2Wo1XuUyLyn6nbGrZQQ==", + "dev": true, + "requires": { + "debug": "^4.1.1", + "through2": "^3.0.1" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "through2": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz", + "integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==", + "dev": true, + "requires": { + "readable-stream": "2 || 3" + } + } + } + }, "rimraf": { "version": "2.6.3", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", @@ -11533,6 +11979,12 @@ } } }, + "walkdir": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.4.1.tgz", + "integrity": "sha512-3eBwRyEln6E1MSzcxcVpQIhRG8Q1jLvEqRmCZqS3dsfXEDR/AhOF4d+jHg1qvDCpYaVRZjENPQyrVxAkQqxPgQ==", + "dev": true + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", diff --git a/garden-service/package.json b/garden-service/package.json index b38c44f0da..86d6b2d8f3 100644 --- a/garden-service/package.json +++ b/garden-service/package.json @@ -123,6 +123,7 @@ "devDependencies": { "@commitlint/cli": "^8.2.0", "@commitlint/config-conventional": "^8.2.0", + "@google-cloud/kms": "^1.5.3", "@types/analytics-node": "^3.1.1", "@types/async-lock": "^1.1.1", "@types/bluebird": "^3.5.29", @@ -134,6 +135,7 @@ "@types/deep-diff": "1.0.0", "@types/dockerode": "^2.5.20", "@types/fs-extra": "^8.0.1", + "@types/google-cloud__kms": "^1.5.0", "@types/hapi__joi": "^15.0.4", "@types/has-ansi": "^3.0.0", "@types/inquirer": "6.5.0", diff --git a/garden-service/src/plugins/kubernetes/api.ts b/garden-service/src/plugins/kubernetes/api.ts index 60e347e13d..633b6a2a59 100644 --- a/garden-service/src/plugins/kubernetes/api.ts +++ b/garden-service/src/plugins/kubernetes/api.ts @@ -83,7 +83,7 @@ const apiTypes: { [key: string]: K8sApiConstructor } = { const crudMap = { Secret: { - type: V1Secret, + cls: new V1Secret(), group: "core", read: "readNamespacedSecret", create: "createNamespacedSecret", @@ -92,7 +92,8 @@ const crudMap = { }, } -type CrudMapType = typeof crudMap +type CrudMap = typeof crudMap +type CrudMapTypes = { [T in keyof CrudMap]: CrudMap[T]["cls"] } export class KubernetesError extends GardenBaseError { type = "kubernetes" @@ -303,22 +304,29 @@ export class KubeApi { return res.body } - async upsert( + async upsert>( kind: K, namespace: string, - obj: KubernetesResource - ): Promise { + obj: O, + log: LogEntry + ) { const api = this[crudMap[kind].group] + const name = obj.metadata.name + + log.debug(`Upserting ${kind} ${namespace}/${name}`) try { - const res = await api[crudMap[kind].read](obj.metadata.name, namespace) - return res.body + await api[crudMap[kind].read](name, namespace) + await api[crudMap[kind].patch](name, namespace, obj) + log.debug(`Patched ${kind} ${namespace}/${name}`) } catch (err) { if (err.code === 404) { try { await api[crudMap[kind].create](namespace, obj) + log.debug(`Created ${kind} ${namespace}/${name}`) } catch (err) { if (err.code === 409) { + log.debug(`Patched ${kind} ${namespace}/${name}`) await api[crudMap[kind].patch](name, namespace, obj) } else { throw err @@ -328,8 +336,6 @@ export class KubeApi { throw err } } - - return obj } /** diff --git a/garden-service/src/plugins/kubernetes/config.ts b/garden-service/src/plugins/kubernetes/config.ts index f7614eb6fd..a7dfa23eaa 100644 --- a/garden-service/src/plugins/kubernetes/config.ts +++ b/garden-service/src/plugins/kubernetes/config.ts @@ -82,7 +82,7 @@ export type ContainerBuildMode = "local-docker" | "cluster-docker" | "kaniko" export type DefaultDeploymentStrategy = "rolling" export type DeploymentStrategy = DefaultDeploymentStrategy | "blue-green" -export interface KubernetesBaseConfig extends ProviderConfig { +export interface KubernetesConfig extends ProviderConfig { buildMode: ContainerBuildMode clusterDocker?: { enableBuildKit?: boolean @@ -90,6 +90,7 @@ export interface KubernetesBaseConfig extends ProviderConfig { context: string defaultHostname?: string defaultUsername?: string + deploymentRegistry?: ContainerRegistryConfig deploymentStrategy?: DeploymentStrategy forceSsl: boolean imagePullSecrets: ProviderSecretRef[] @@ -107,10 +108,6 @@ export interface KubernetesBaseConfig extends ProviderConfig { _systemServices: string[] } -export interface KubernetesConfig extends KubernetesBaseConfig { - deploymentRegistry?: ContainerRegistryConfig -} - export type KubernetesProvider = Provider export type KubernetesPluginContext = PluginContext diff --git a/garden-service/src/plugins/kubernetes/constants.ts b/garden-service/src/plugins/kubernetes/constants.ts index aa4b3db6d4..f2133e5be0 100644 --- a/garden-service/src/plugins/kubernetes/constants.ts +++ b/garden-service/src/plugins/kubernetes/constants.ts @@ -11,3 +11,6 @@ export const CLUSTER_REGISTRY_PORT = 5000 export const CLUSTER_REGISTRY_DEPLOYMENT_NAME = "garden-docker-registry" export const MAX_CONFIGMAP_DATA_SIZE = 1024 * 1024 // max ConfigMap data size is 1MB export const MAX_RUN_RESULT_OUTPUT_LENGTH = 900 * 1024 // max ConfigMap data size is 1MB, so 900kB gives enough margin + +export const dockerAuthSecretName = "builder-docker-config" +export const dockerAuthSecretKey = ".dockerconfigjson" diff --git a/garden-service/src/plugins/kubernetes/container/build.ts b/garden-service/src/plugins/kubernetes/container/build.ts index 8ef77c142c..741692478c 100644 --- a/garden-service/src/plugins/kubernetes/container/build.ts +++ b/garden-service/src/plugins/kubernetes/container/build.ts @@ -14,7 +14,7 @@ import { buildContainerModule, getContainerBuildStatus, getDockerBuildFlags } fr import { GetBuildStatusParams, BuildStatus } from "../../../types/plugin/module/getBuildStatus" import { BuildModuleParams, BuildResult } from "../../../types/plugin/module/build" import { millicpuToString, megabytesToString, getRunningPodInDeployment } from "../util" -import { RSYNC_PORT } from "../constants" +import { RSYNC_PORT, dockerAuthSecretName, dockerAuthSecretKey } from "../constants" import { posix, resolve } from "path" import { KubeApi } from "../api" import { kubectl } from "../kubectl" @@ -329,6 +329,21 @@ async function runKaniko({ provider, log, module, args, outputStream }: RunKanik namespace: systemNamespace, spec: { shareProcessNamespace: true, + volumes: [ + // Mount the build sync volume, to get the build context from. + { + name: syncDataVolumeName, + persistentVolumeClaim: { claimName: syncDataVolumeName }, + }, + // Mount the docker auth secret, so Kaniko can pull from private registries. + { + name: dockerAuthSecretName, + secret: { + secretName: dockerAuthSecretName, + items: [{ key: dockerAuthSecretKey, path: "config.json" }], + }, + }, + ], containers: [ { name: "kaniko", @@ -339,6 +354,11 @@ async function runKaniko({ provider, log, module, args, outputStream }: RunKanik name: syncDataVolumeName, mountPath: "/garden-build", }, + { + name: dockerAuthSecretName, + mountPath: "/kaniko/.docker", + readOnly: true, + }, ], resources: { limits: { @@ -377,12 +397,6 @@ async function runKaniko({ provider, log, module, args, outputStream }: RunKanik ], }, ], - volumes: [ - { - name: syncDataVolumeName, - persistentVolumeClaim: { claimName: syncDataVolumeName }, - }, - ], }, }) diff --git a/garden-service/src/plugins/kubernetes/container/deployment.ts b/garden-service/src/plugins/kubernetes/container/deployment.ts index f330101064..5578f55e82 100644 --- a/garden-service/src/plugins/kubernetes/container/deployment.ts +++ b/garden-service/src/plugins/kubernetes/container/deployment.ts @@ -194,7 +194,7 @@ export async function createContainerManifests( const { production } = ctx const namespace = await getAppNamespace(k8sCtx, log, provider) const api = await KubeApi.factory(log, provider) - const ingresses = await createIngressResources(api, provider, namespace, service) + const ingresses = await createIngressResources(api, provider, namespace, service, log) const workload = await createWorkloadResource({ provider, service, diff --git a/garden-service/src/plugins/kubernetes/container/ingress.ts b/garden-service/src/plugins/kubernetes/container/ingress.ts index 51eb709e46..63aaf4d971 100644 --- a/garden-service/src/plugins/kubernetes/container/ingress.ts +++ b/garden-service/src/plugins/kubernetes/container/ingress.ts @@ -18,6 +18,7 @@ import { ensureSecret } from "../secrets" import { getHostnamesFromPem } from "../../../util/tls" import { KubernetesResource } from "../types" import { V1Secret } from "@kubernetes/client-node" +import { LogEntry } from "../../../logger/log-entry" interface ServiceIngressWithCert extends ServiceIngress { spec: ContainerIngressSpec @@ -30,7 +31,8 @@ export async function createIngressResources( api: KubeApi, provider: KubernetesProvider, namespace: string, - service: ContainerService + service: ContainerService, + log: LogEntry ) { if (service.spec.ingresses.length === 0) { return [] @@ -72,7 +74,7 @@ export async function createIngressResources( if (!!cert) { // make sure the TLS secrets exist in this namespace - await ensureSecret(api, cert.secretRef, namespace) + await ensureSecret(api, cert.secretRef, namespace, log) spec.tls = [ { diff --git a/garden-service/src/plugins/kubernetes/init.ts b/garden-service/src/plugins/kubernetes/init.ts index 713e6b6bf4..f0b783bcce 100644 --- a/garden-service/src/plugins/kubernetes/init.ts +++ b/garden-service/src/plugins/kubernetes/init.ts @@ -8,20 +8,15 @@ import { KubeApi, KubernetesError } from "./api" import { getAppNamespace, prepareNamespaces, deleteNamespaces, getMetadataNamespace } from "./namespace" -import { KubernetesPluginContext, KubernetesConfig } from "./config" +import { KubernetesPluginContext, KubernetesConfig, KubernetesProvider } from "./config" import { checkTillerStatus, installTiller } from "./helm/tiller" -import { - prepareSystemServices, - getSystemServiceStatus, - getSystemGarden, - systemNamespaceUpToDate, -} from "./system" +import { prepareSystemServices, getSystemServiceStatus, getSystemGarden, systemNamespaceUpToDate } from "./system" import { GetEnvironmentStatusParams, EnvironmentStatus } from "../../types/plugin/provider/getEnvironmentStatus" import { PrepareEnvironmentParams, PrepareEnvironmentResult } from "../../types/plugin/provider/prepareEnvironment" import { CleanupEnvironmentParams } from "../../types/plugin/provider/cleanupEnvironment" import { millicpuToString, megabytesToString } from "./util" import chalk from "chalk" -import { deline } from "../../util/string" +import { deline, dedent } from "../../util/string" import { combineStates, ServiceStatusMap } from "../../types/service" import { setupCertManager, @@ -30,10 +25,23 @@ import { getCertificateName, } from "./integrations/cert-manager" import { ConfigurationError } from "../../exceptions" +import Bluebird from "bluebird" +import { readSecret } from "./secrets" +import { extend } from "lodash" +import { dockerAuthSecretName, dockerAuthSecretKey } from "./constants" +import { V1Secret } from "@kubernetes/client-node" +import { KubernetesResource } from "./types" +import { compareDeployedResources } from "./status/status" // Note: We need to increment a version number here if we ever make breaking changes to the NFS provisioner StatefulSet const nfsStorageClassVersion = 2 +const dockerAuthSecretType = "kubernetes.io/dockerconfigjson" +const dockerAuthDocsLink = ` +See https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ for how to create +a registry auth secret. +` + /** * Performs the following actions to check environment status: * 1. Checks Tiller status in the project namespace @@ -139,6 +147,15 @@ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusPar const contextForLog = `Checking Garden system service status for plugin "${ctx.provider.name}"` const sysNamespaceUpToDate = await systemNamespaceUpToDate(api, log, systemNamespace, contextForLog) + // Check if builder auth secret is up-to-date + let secretsUpToDate = true + + if (provider.config.buildMode !== "local-docker") { + const authSecret = await prepareDockerAuth(api, provider) + const comparison = await compareDeployedResources(k8sCtx, api, systemNamespace, [authSecret], log) + secretsUpToDate = comparison.state === "ready" + } + // Get system service statuses const systemServiceStatus = await getSystemServiceStatus({ ctx: k8sCtx, @@ -148,7 +165,7 @@ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusPar serviceNames: systemServiceNames, }) - if (!sysNamespaceUpToDate || systemServiceStatus.state !== "ready") { + if (!sysNamespaceUpToDate || !secretsUpToDate || systemServiceStatus.state !== "ready") { result.ready = false detail.systemReady = false } @@ -254,6 +271,12 @@ export async function prepareSystem({ await installTiller({ ctx: sysCtx, provider: sysCtx.provider, log, force }) + if (provider.config.buildMode !== "local-docker") { + const api = await KubeApi.factory(log, provider) + const authSecret = await prepareDockerAuth(api, provider) + await api.upsert("Secret", systemNamespace, authSecret, log) + } + // We need to install the NFS provisioner separately, so that we can optionally install it // FIXME: when we've added an `enabled` field, we should get rid of this special case if (systemServiceNames.includes("nfs-provisioner")) { @@ -345,3 +368,81 @@ export function getRegistryHostname(config: KubernetesConfig) { const systemNamespace = config.gardenSystemNamespace return `garden-docker-registry.${systemNamespace}.svc.cluster.local` } +async function prepareDockerAuth(api: KubeApi, provider: KubernetesProvider): Promise> { + // Read all configured imagePullSecrets and combine into a docker config file to use in the in-cluster builders. + const auths: { [name: string]: any } = {} + + await Bluebird.map(provider.config.imagePullSecrets, async (secretRef) => { + const secret = await readSecret(api, secretRef) + + if (secret.type !== dockerAuthSecretType) { + throw new ConfigurationError( + dedent` + Configured imagePullSecret '${secret.metadata.name}' does not appear to be a valid registry secret, because + it does not have \`type: ${dockerAuthSecretType}\`. + ${dockerAuthDocsLink} + `, + { secretRef } + ) + } + + // Decode the secret + const encoded = secret.data && secret.data![dockerAuthSecretKey] + + if (!encoded) { + throw new ConfigurationError( + dedent` + Configured imagePullSecret '${secret.metadata.name}' does not appear to be a valid registry secret, because + it does not contain a ${dockerAuthSecretKey} key. + ${dockerAuthDocsLink} + `, + { secretRef } + ) + } + + let decoded: any + + try { + decoded = JSON.parse(Buffer.from(encoded, "base64").toString()) + } catch (err) { + throw new ConfigurationError( + dedent` + Could not parse configured imagePullSecret '${secret.metadata.name}' as a JSON docker authentication file: + ${err.message}. + ${dockerAuthDocsLink} + `, + { secretRef } + ) + } + + if (!decoded.auths) { + throw new ConfigurationError( + dedent` + Could not parse configured imagePullSecret '${secret.metadata.name}' as a valid docker authentication file, + because it is missing an "auths" key. + ${dockerAuthDocsLink} + `, + { secretRef } + ) + } + + extend(auths, decoded.auths) + }) + + const config = { auths } + + // Store the config as a Secret (overwriting if necessary) + const systemNamespace = provider.config.gardenSystemNamespace + + return { + apiVersion: "v1", + kind: "Secret", + metadata: { + name: dockerAuthSecretName, + namespace: systemNamespace, + }, + data: { + [dockerAuthSecretKey]: Buffer.from(JSON.stringify(config)).toString("base64"), + }, + } +} diff --git a/garden-service/src/plugins/kubernetes/kubernetes.ts b/garden-service/src/plugins/kubernetes/kubernetes.ts index 4016052a2a..6e5812d36f 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes.ts @@ -31,7 +31,7 @@ import { kubernetesModuleSpecSchema } from "./kubernetes-module/config" import { helmModuleSpecSchema, helmModuleOutputsSchema } from "./helm/config" import { isNumber } from "util" import chalk from "chalk" -import pluralize = require("pluralize") +import pluralize from "pluralize" import { getSystemMetadataNamespaceName } from "./system" export async function configureProvider({ @@ -47,6 +47,10 @@ export async function configureProvider({ if (config.setupIngressController === "nginx") { config._systemServices.push("ingress-controller", "default-backend") + + if (!config.ingressClass) { + config.ingressClass = "nginx" + } } if (config.buildMode === "cluster-docker" || config.buildMode === "kaniko") { diff --git a/garden-service/src/plugins/kubernetes/local/config.ts b/garden-service/src/plugins/kubernetes/local/config.ts index 43f7678316..3831ac7a36 100644 --- a/garden-service/src/plugins/kubernetes/local/config.ts +++ b/garden-service/src/plugins/kubernetes/local/config.ts @@ -7,22 +7,23 @@ */ import Bluebird from "bluebird" -import { KubernetesBaseConfig, kubernetesConfigBase, k8sContextSchema } from "../config" +import { KubernetesConfig, kubernetesConfigBase, k8sContextSchema } from "../config" import { ConfigureProviderParams } from "../../../types/plugin/provider/configureProvider" import { joiProviderName, joi } from "../../../config/common" import { getKubeConfig } from "../api" import { configureMicrok8sAddons } from "./microk8s" import { setMinikubeDockerEnv } from "./minikube" -import { ContainerRegistryConfig } from "../../container/config" import { exec } from "../../../util/util" +import { remove } from "lodash" // TODO: split this into separate plugins to handle Docker for Mac and Minikube // note: this is in order of preference, in case neither is set as the current kubectl context // and none is explicitly configured in the garden.yml const supportedContexts = ["docker-for-desktop", "microk8s", "minikube"] +const nginxServices = ["ingress-controller", "default-backend"] -export interface LocalKubernetesConfig extends KubernetesBaseConfig { +export interface LocalKubernetesConfig extends KubernetesConfig { setupIngressController: string | null } @@ -45,17 +46,14 @@ export const configSchema = kubernetesConfigBase }) .description("The provider configuration for the local-kubernetes plugin.") -export async function configureProvider({ config, log, projectName }: ConfigureProviderParams) { - let context = config.context - let defaultHostname = config.defaultHostname - let deploymentRegistry: ContainerRegistryConfig | undefined = undefined +export async function configureProvider(params: ConfigureProviderParams) { + const { base, log, projectName } = params + let { config } = await base!(params) - const namespace = config.namespace || projectName - const _systemServices: string[] = [] + const namespace = config.namespace! + const _systemServices = config._systemServices - const deploymentStrategy = config.deploymentStrategy || "rolling" - - if (!context) { + if (!config.context) { // automatically detect supported kubectl context if not explicitly configured // create dummy provider with just enough info needed for the getKubeConfig function const provider = { @@ -70,101 +68,76 @@ export async function configureProvider({ config, log, projectName }: ConfigureP if (currentContext && supportedContexts.includes(currentContext)) { // prefer current context if set and supported - context = currentContext - log.debug({ section: config.name, msg: `Using current context: ${context}` }) + config.context = currentContext + log.debug({ section: config.name, msg: `Using current context: ${config.context}` }) } else { - const availableContexts = kubeConfig.contexts.map((c) => c.name) + const availableContexts = kubeConfig.contexts.map((c: any) => c.name) for (const supportedContext of supportedContexts) { if (availableContexts.includes(supportedContext)) { - context = supportedContext - log.debug({ section: config.name, msg: `Using detected context: ${context}` }) + config.context = supportedContext + log.debug({ section: config.name, msg: `Using detected context: ${config.context}` }) break } } } - if (!context && kubeConfig.contexts.length > 0) { - context = kubeConfig.contexts[0].name - log.debug({ section: config.name, msg: `No kubectl context auto-detected, using first available: ${context}` }) + if (!config.context && kubeConfig.contexts.length > 0) { + config.context = kubeConfig.contexts[0].name + log.debug({ + section: config.name, + msg: `No kubectl context auto-detected, using first available: ${config.context}`, + }) } } - if (!context) { - context = supportedContexts[0] - log.debug({ section: config.name, msg: `No kubectl context configured, using default: ${context}` }) + if (!config.context) { + config.context = supportedContexts[0] + log.debug({ section: config.name, msg: `No kubectl context configured, using default: ${config.context}` }) } - if (context === "minikube") { + if (config.context === "minikube") { const initCmds = [ ["config", "set", "WantUpdateNotification", "false"], ["addons", "enable", "dashboard"], ] await Bluebird.map(initCmds, async (cmd) => exec("minikube", cmd)) - if (!defaultHostname) { + if (!config.defaultHostname) { // use the nip.io service to give a hostname to the instance, if none is explicitly configured const { stdout } = await exec("minikube", ["ip"]) - defaultHostname = `${projectName}.${stdout}.nip.io` + config.defaultHostname = `${projectName}.${stdout}.nip.io` } if (config.setupIngressController === "nginx") { log.debug("Using minikube's ingress addon") await exec("minikube", ["addons", "enable", "ingress"]) + remove(_systemServices, (s) => nginxServices.includes(s)) } await setMinikubeDockerEnv() - } else if (context === "microk8s") { + } else if (config.context === "microk8s") { const addons = ["dns", "dashboard", "registry", "storage"] if (config.setupIngressController === "nginx") { log.debug("Using microk8s's ingress addon") addons.push("ingress") + remove(_systemServices, (s) => nginxServices.includes(s)) } await configureMicrok8sAddons(log, addons) // Need to push to the built-in registry - deploymentRegistry = { + config.deploymentRegistry = { hostname: "localhost:32000", namespace, } } else { _systemServices.push("kubernetes-dashboard") - // Install nginx on init - if (config.setupIngressController === "nginx") { - _systemServices.push("ingress-controller", "default-backend") - } - } - - if (!defaultHostname) { - defaultHostname = `${projectName}.local.app.garden` } - const ingressClass = config.ingressClass || config.setupIngressController || undefined - - config = { - // Setting the name to kubernetes, so that plugins that depend on kubernetes can reference it. - name: config.name, - buildMode: config.buildMode, - context, - defaultHostname, - deploymentRegistry, - deploymentStrategy, - forceSsl: false, - gardenSystemNamespace: config.gardenSystemNamespace, - imagePullSecrets: config.imagePullSecrets, - ingressHttpPort: 80, - ingressHttpsPort: 443, - ingressClass, - namespace, - registryProxyTolerations: config.registryProxyTolerations, - resources: config.resources, - storage: config.storage, - setupIngressController: config.setupIngressController, - tlsCertificates: config.tlsCertificates, - certManager: config.certManager, - _systemServices, + if (!config.defaultHostname) { + config.defaultHostname = `${projectName}.local.app.garden` } return { config } diff --git a/garden-service/src/plugins/kubernetes/secrets.ts b/garden-service/src/plugins/kubernetes/secrets.ts index 956274188a..ebb553e41d 100644 --- a/garden-service/src/plugins/kubernetes/secrets.ts +++ b/garden-service/src/plugins/kubernetes/secrets.ts @@ -6,8 +6,6 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { V1Secret } from "@kubernetes/client-node" - import { KubeApi } from "./api" import { ProviderSecretRef, KubernetesPluginContext } from "./config" import { ConfigurationError } from "../../exceptions" @@ -15,8 +13,8 @@ import { getMetadataNamespace } from "./namespace" import { GetSecretParams } from "../../types/plugin/provider/getSecret" import { SetSecretParams } from "../../types/plugin/provider/setSecret" import { DeleteSecretParams } from "../../types/plugin/provider/deleteSecret" -import { KubernetesResource } from "./types" import { pick } from "lodash" +import { LogEntry } from "../../logger/log-entry" export async function getSecret({ ctx, log, key }: GetSecretParams) { const k8sCtx = ctx @@ -86,13 +84,11 @@ export async function deleteSecret({ ctx, log, key }: DeleteSecretParams) { } /** - * Make sure the specified secret exists in the target namespace, copying it if necessary. + * Read the specified secret ref from the cluster. */ -export async function ensureSecret(api: KubeApi, secretRef: ProviderSecretRef, targetNamespace: string) { - let secret: KubernetesResource - +export async function readSecret(api: KubeApi, secretRef: ProviderSecretRef) { try { - secret = await api.core.readNamespacedSecret(secretRef.name, secretRef.namespace) + return await api.core.readNamespacedSecret(secretRef.name, secretRef.namespace) } catch (err) { if (err.code === 404) { throw new ConfigurationError( @@ -106,6 +102,13 @@ export async function ensureSecret(api: KubeApi, secretRef: ProviderSecretRef, t throw err } } +} + +/** + * Make sure the specified secret exists in the target namespace, copying it if necessary. + */ +export async function ensureSecret(api: KubeApi, secretRef: ProviderSecretRef, targetNamespace: string, log: LogEntry) { + const secret = await readSecret(api, secretRef) if (secretRef.namespace === targetNamespace) { return @@ -117,5 +120,5 @@ export async function ensureSecret(api: KubeApi, secretRef: ProviderSecretRef, t namespace: targetNamespace, } - await api.upsert("Secret", targetNamespace, secret) + await api.upsert("Secret", targetNamespace, secret, log) } diff --git a/garden-service/static/kubernetes/system/docker-daemon/templates/deployment.yaml b/garden-service/static/kubernetes/system/docker-daemon/templates/deployment.yaml index 34f5fafc16..8ec8753f2e 100644 --- a/garden-service/static/kubernetes/system/docker-daemon/templates/deployment.yaml +++ b/garden-service/static/kubernetes/system/docker-daemon/templates/deployment.yaml @@ -29,12 +29,12 @@ spec: - name: garden-build-sync persistentVolumeClaim: claimName: {{ .Values.buildSync.volume.name }} - # - name: garden-registry-tls - # secret: - # secretName: foo - # items: - # - key: tls.crt - # path: localhost:5000/ca.crt + - name: docker-config + secret: + secretName: builder-docker-config + items: + - key: .dockerconfigjson + path: config.json containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -60,9 +60,8 @@ spec: mountPath: /var/lib/docker - name: garden-build-sync mountPath: /garden-build - # Need to mount the registry cert so that the daemon trusts it - # - name: garden-registry-tls - # mountPath: /etc/docker/certs.d + - name: docker-config + mountPath: /root/.docker resources: {{- toYaml .Values.resources | nindent 12 }} - name: proxy diff --git a/garden-service/test/data/test-projects/container/garden.yml b/garden-service/test/data/test-projects/container/garden.yml index 7732622541..01cba82ae2 100644 --- a/garden-service/test/data/test-projects/container/garden.yml +++ b/garden-service/test/data/test-projects/container/garden.yml @@ -1,4 +1,31 @@ kind: Project -name: container-artifacts -providers: - - name: local-kubernetes +name: container +environments: + - name: local + providers: + - name: local-kubernetes + - name: cluster-docker + providers: + - &clusterDocker + name: local-kubernetes + buildMode: cluster-docker + imagePullSecrets: + # Note: We populate this secret in the test code + - name: test-docker-auth + # This is currently necessary for Docker Desktop, and potentially other local K8s distros + # Note: It is _terribly_ slow, so should only be used for testing. + storage: + builder: + storageClass: garden-system-nfs-v2 + - name: cluster-docker-buildkit + providers: + - <<: *clusterDocker + clusterDocker: + enableBuildKit: true + - name: cluster-docker-auth + providers: + - <<: *clusterDocker + - name: kaniko + providers: + - <<: *clusterDocker + buildMode: kaniko diff --git a/garden-service/test/data/test-projects/container/inaccessible-base/Dockerfile b/garden-service/test/data/test-projects/container/inaccessible-base/Dockerfile new file mode 100644 index 0000000000..acc65af365 --- /dev/null +++ b/garden-service/test/data/test-projects/container/inaccessible-base/Dockerfile @@ -0,0 +1,3 @@ +FROM edvald/test:0.1.0 + +RUN echo foo diff --git a/garden-service/test/data/test-projects/container/inaccessible-base/garden.yml b/garden-service/test/data/test-projects/container/inaccessible-base/garden.yml new file mode 100644 index 0000000000..6ab5a14f6f --- /dev/null +++ b/garden-service/test/data/test-projects/container/inaccessible-base/garden.yml @@ -0,0 +1,4 @@ +kind: Module +name: inaccessible-base +description: Test module for attempting to pull from inaccessible registries +type: container diff --git a/garden-service/test/data/test-projects/container/private-base/Dockerfile b/garden-service/test/data/test-projects/container/private-base/Dockerfile new file mode 100644 index 0000000000..bbd5f62d6d --- /dev/null +++ b/garden-service/test/data/test-projects/container/private-base/Dockerfile @@ -0,0 +1,3 @@ +FROM gardendev/garden-pull-test:0.1.0 + +ADD foo.txt /foo.txt diff --git a/garden-service/test/data/test-projects/container/private-base/foo.txt b/garden-service/test/data/test-projects/container/private-base/foo.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/garden-service/test/data/test-projects/container/private-base/garden.yml b/garden-service/test/data/test-projects/container/private-base/garden.yml new file mode 100644 index 0000000000..50d8784742 --- /dev/null +++ b/garden-service/test/data/test-projects/container/private-base/garden.yml @@ -0,0 +1,4 @@ +kind: Module +name: private-base +description: Test module for pulling from private registries +type: container diff --git a/garden-service/test/data/test-projects/container/simple-service/Dockerfile b/garden-service/test/data/test-projects/container/simple-service/Dockerfile index 3e3bcd9bb6..254c6c1bdb 100644 --- a/garden-service/test/data/test-projects/container/simple-service/Dockerfile +++ b/garden-service/test/data/test-projects/container/simple-service/Dockerfile @@ -1,7 +1,5 @@ -FROM busybox +FROM busybox:1.31.1 COPY main . ENTRYPOINT ./main - -EXPOSE 8080 \ No newline at end of file diff --git a/garden-service/test/data/test-projects/container/simple/garden.yml b/garden-service/test/data/test-projects/container/simple/garden.yml index ef7622fcd0..b82f4bf3b2 100644 --- a/garden-service/test/data/test-projects/container/simple/garden.yml +++ b/garden-service/test/data/test-projects/container/simple/garden.yml @@ -1,7 +1,7 @@ kind: Module name: simple type: container -image: busybox +image: busybox:1.31.1 services: - name: echo-service command: [sh, -c, "echo ok"] diff --git a/garden-service/test/helpers.ts b/garden-service/test/helpers.ts index 2ccb5f23ea..04c765996a 100644 --- a/garden-service/test/helpers.ts +++ b/garden-service/test/helpers.ts @@ -23,7 +23,7 @@ import { RegisterPluginParam, ModuleAndRuntimeActionHandlers, } from "../src/types/plugin/plugin" -import { Garden, GardenParams } from "../src/garden" +import { Garden, GardenParams, GardenOpts } from "../src/garden" import { ModuleConfig } from "../src/config/module" import { mapValues, fromPairs } from "lodash" import { ModuleVersion } from "../src/vcs/vcs" @@ -329,16 +329,13 @@ export class TestGarden extends Garden { } } -export const makeTestGarden = async ( - projectRoot: string, - { extraPlugins, gardenDirPath }: { extraPlugins?: RegisterPluginParam[]; gardenDirPath?: string } = {} -): Promise => { - const plugins = [...testPlugins, ...(extraPlugins || [])] - return TestGarden.factory(projectRoot, { plugins, gardenDirPath }) +export const makeTestGarden = async (projectRoot: string, opts: GardenOpts = {}): Promise => { + const plugins = [...testPlugins, ...(opts.plugins || [])] + return TestGarden.factory(projectRoot, { ...opts, plugins }) } export const makeTestGardenA = async (extraPlugins: RegisterPluginParam[] = []) => { - return makeTestGarden(projectRootA, { extraPlugins }) + return makeTestGarden(projectRootA, { plugins: extraPlugins }) } export function stubAction( diff --git a/garden-service/test/integ/helpers.ts b/garden-service/test/integ/helpers.ts new file mode 100644 index 0000000000..c0a63098cb --- /dev/null +++ b/garden-service/test/integ/helpers.ts @@ -0,0 +1,20 @@ +import kms from "@google-cloud/kms" +import { readFile } from "fs-extra" + +const projectId = "garden-dev-200012" +const keyRingId = "dev" +const cryptoKeyId = "dev" +const locationId = "global" + +/** + * Decrypt a secret file, encrypted with our Google Cloud KMS key. + */ +export async function decryptSecretFile(path: string) { + const client = new kms.KeyManagementServiceClient() + + const name = client.cryptoKeyPath(projectId, locationId, keyRingId, cryptoKeyId) + const ciphertext = await readFile(path) + const [result] = await client.decrypt({ name, ciphertext }) + + return result.plaintext +} diff --git a/garden-service/test/integ/src/plugins/kubernetes/container/build.ts b/garden-service/test/integ/src/plugins/kubernetes/container/build.ts new file mode 100644 index 0000000000..bb92c78ddf --- /dev/null +++ b/garden-service/test/integ/src/plugins/kubernetes/container/build.ts @@ -0,0 +1,207 @@ +import { getDataDir, makeTestGarden, expectError } from "../../../../../helpers" +import { Garden } from "../../../../../../src/garden" +import { ConfigGraph } from "../../../../../../src/config-graph" +import { k8sBuildContainer } from "../../../../../../src/plugins/kubernetes/container/build" +import { PluginContext } from "../../../../../../src/plugin-context" +import { clusterInit } from "../../../../../../src/plugins/kubernetes/commands/cluster-init" +import { KubernetesProvider } from "../../../../../../src/plugins/kubernetes/config" +import { decryptSecretFile } from "../../../../helpers" +import { GARDEN_SERVICE_ROOT } from "../../../../../../src/constants" +import { resolve } from "path" +import { KubeApi } from "../../../../../../src/plugins/kubernetes/api" +import { expect } from "chai" + +describe("k8sBuildContainer", () => { + let garden: Garden + let graph: ConfigGraph + let provider: KubernetesProvider + let ctx: PluginContext + + let initialized = false + + const root = getDataDir("test-projects", "container") + + after(async () => { + if (garden) { + await garden.close() + } + }) + + const init = async (environmentName: string) => { + if (!initialized && environmentName !== "local") { + // Load the test authentication for private registries + const authSecret = JSON.parse( + (await decryptSecretFile(resolve(GARDEN_SERVICE_ROOT, "..", "secrets", "test-docker-auth.json"))).toString() + ) + const api = await KubeApi.factory(garden.log, provider) + await api.upsert("Secret", "default", authSecret, garden.log) + } + + garden = await makeTestGarden(root, { environmentName }) + graph = await garden.getConfigGraph() + provider = await garden.resolveProvider("local-kubernetes") + ctx = garden.getPluginContext(provider) + + // We only need to run the cluster-init flow once, because the configurations are compatible + if (!initialized && environmentName !== "local") { + // Run cluster-init + await clusterInit.handler({ ctx, log: garden.log }) + initialized = true + } + } + + context("local mode", () => { + before(async () => { + await init("local") + }) + + it("should build a simple container", async () => { + const module = await graph.getModule("simple-service") + await garden.buildDir.syncFromSrc(module, garden.log) + + await k8sBuildContainer({ + ctx, + log: garden.log, + module, + }) + }) + }) + + context("cluster-docker mode", () => { + before(async () => { + await init("cluster-docker") + }) + + it("should build a simple container", async () => { + const module = await graph.getModule("simple-service") + await garden.buildDir.syncFromSrc(module, garden.log) + + await k8sBuildContainer({ + ctx, + log: garden.log, + module, + }) + }) + + it("should support pulling from private registries", async () => { + const module = await graph.getModule("private-base") + await garden.buildDir.syncFromSrc(module, garden.log) + + await k8sBuildContainer({ + ctx, + log: garden.log, + module, + }) + }) + + it("should throw if attempting to pull from private registry without access", async () => { + const module = await graph.getModule("inaccessible-base") + await garden.buildDir.syncFromSrc(module, garden.log) + + await expectError( + () => + k8sBuildContainer({ + ctx, + log: garden.log, + module, + }), + (err) => { + expect(err.message).to.include("pull access denied") + } + ) + }) + }) + + context("cluster-docker mode with BuildKit", () => { + before(async () => { + await init("cluster-docker-buildkit") + }) + + it("should build a simple container", async () => { + const module = await graph.getModule("simple-service") + await garden.buildDir.syncFromSrc(module, garden.log) + + const result = await k8sBuildContainer({ + ctx, + log: garden.log, + module, + }) + + // Make sure we're actually using BuildKit + expect(result.buildLog!).to.include("load build definition from Dockerfile") + }) + + it("should support pulling from private registries", async () => { + const module = await graph.getModule("private-base") + await garden.buildDir.syncFromSrc(module, garden.log) + + await k8sBuildContainer({ + ctx, + log: garden.log, + module, + }) + }) + + it("should throw if attempting to pull from private registry without access", async () => { + const module = await graph.getModule("inaccessible-base") + await garden.buildDir.syncFromSrc(module, garden.log) + + await expectError( + () => + k8sBuildContainer({ + ctx, + log: garden.log, + module, + }), + (err) => { + expect(err.message).to.include("pull access denied") + } + ) + }) + }) + + context("kaniko mode", () => { + before(async () => { + await init("kaniko") + }) + + it("should build a simple container", async () => { + const module = await graph.getModule("simple-service") + await garden.buildDir.syncFromSrc(module, garden.log) + + await k8sBuildContainer({ + ctx, + log: garden.log, + module, + }) + }) + + it("should support pulling from private registries", async () => { + const module = await graph.getModule("private-base") + await garden.buildDir.syncFromSrc(module, garden.log) + + await k8sBuildContainer({ + ctx, + log: garden.log, + module, + }) + }) + + it("should throw if attempting to pull from private registry without access", async () => { + const module = await graph.getModule("inaccessible-base") + await garden.buildDir.syncFromSrc(module, garden.log) + + await expectError( + () => + k8sBuildContainer({ + ctx, + log: garden.log, + module, + }), + (err) => { + expect(err.message).to.include("UNAUTHORIZED") + } + ) + }) + }) +}) diff --git a/garden-service/test/integ/src/plugins/kubernetes/util.ts b/garden-service/test/integ/src/plugins/kubernetes/util.ts index 541b4052b8..8925b4ffd3 100644 --- a/garden-service/test/integ/src/plugins/kubernetes/util.ts +++ b/garden-service/test/integ/src/plugins/kubernetes/util.ts @@ -47,14 +47,14 @@ describe("util", () => { provider, service, runtimeContext: emptyRuntimeContext, - namespace: "container-artifacts", + namespace: "container", enableHotReload: false, log: garden.log, production: false, }) await garden.processTasks([deployTask], { throwOnError: true }) - const pods = await getWorkloadPods(api, "container-artifacts", resource) + const pods = await getWorkloadPods(api, "container", resource) const services = flatten(pods.map((pod) => pod.spec.containers.map((container) => container.name))) expect(services).to.eql(["simple-service"]) }) diff --git a/garden-service/test/mocha.integ.opts b/garden-service/test/mocha.integ.opts index e93d8fe906..22d79ab208 100644 --- a/garden-service/test/mocha.integ.opts +++ b/garden-service/test/mocha.integ.opts @@ -5,7 +5,7 @@ --exclude test/,src/,.garden --watch-extensions js,json,pegjs --reporter spec ---timeout 90000 +--timeout 300000 --exit build/test/setup.js build/test/integ/**/*.js diff --git a/garden-service/test/unit/src/plugins/container/container.ts b/garden-service/test/unit/src/plugins/container/container.ts index 0c25747cc7..3e08ec6d43 100644 --- a/garden-service/test/unit/src/plugins/container/container.ts +++ b/garden-service/test/unit/src/plugins/container/container.ts @@ -66,7 +66,7 @@ describe("plugins.container", () => { let log: LogEntry beforeEach(async () => { - garden = await makeTestGarden(projectRoot, { extraPlugins: [gardenPlugin] }) + garden = await makeTestGarden(projectRoot, { plugins: [gardenPlugin] }) log = garden.log const provider = await garden.resolveProvider("container") ctx = garden.getPluginContext(provider) diff --git a/garden-service/test/unit/src/plugins/container/helpers.ts b/garden-service/test/unit/src/plugins/container/helpers.ts index 6f704f86f3..a6d0e18622 100644 --- a/garden-service/test/unit/src/plugins/container/helpers.ts +++ b/garden-service/test/unit/src/plugins/container/helpers.ts @@ -54,7 +54,7 @@ describe("containerHelpers", () => { let log: LogEntry beforeEach(async () => { - garden = await makeTestGarden(projectRoot, { extraPlugins: [gardenPlugin] }) + garden = await makeTestGarden(projectRoot, { plugins: [gardenPlugin] }) log = garden.log const provider = await garden.resolveProvider("container") ctx = garden.getPluginContext(provider) diff --git a/garden-service/test/unit/src/plugins/exec.ts b/garden-service/test/unit/src/plugins/exec.ts index e2809e52c1..eecc9876ae 100644 --- a/garden-service/test/unit/src/plugins/exec.ts +++ b/garden-service/test/unit/src/plugins/exec.ts @@ -24,7 +24,7 @@ describe("exec plugin", () => { let log: LogEntry beforeEach(async () => { - garden = await makeTestGarden(projectRoot, { extraPlugins: [gardenPlugin] }) + garden = await makeTestGarden(projectRoot, { plugins: [gardenPlugin] }) graph = await garden.getConfigGraph() log = garden.log await garden.clearBuilds() diff --git a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts index 749156be1d..25d6d6147d 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts @@ -333,7 +333,7 @@ describe("createIngressResources", () => { }) beforeEach(async () => { - garden = await makeTestGarden(projectRoot, { extraPlugins: [gardenPlugin] }) + garden = await makeTestGarden(projectRoot, { plugins: [gardenPlugin] }) td.replace(garden.buildDir, "syncDependencyProducts", () => null) @@ -424,7 +424,7 @@ describe("createIngressResources", () => { }) const api = await getKubeApi(basicProvider) - const ingresses = await createIngressResources(api, basicProvider, namespace, service) + const ingresses = await createIngressResources(api, basicProvider, namespace, service, garden.log) expect(ingresses).to.eql([ { @@ -468,7 +468,7 @@ describe("createIngressResources", () => { }) const api = await getKubeApi(basicProvider) - const ingresses = await createIngressResources(api, basicProvider, namespace, service) + const ingresses = await createIngressResources(api, basicProvider, namespace, service, garden.log) expect(ingresses).to.eql([ { @@ -521,7 +521,7 @@ describe("createIngressResources", () => { ) const api = await getKubeApi(basicProvider) - const ingresses = await createIngressResources(api, basicProvider, namespace, service) + const ingresses = await createIngressResources(api, basicProvider, namespace, service, garden.log) expect(ingresses).to.eql([ { @@ -595,9 +595,9 @@ describe("createIngressResources", () => { }) const api = await getKubeApi(singleTlsProvider) - const ingresses = await createIngressResources(api, singleTlsProvider, namespace, service) + const ingresses = await createIngressResources(api, singleTlsProvider, namespace, service, garden.log) - td.verify(api.upsert("Secret", namespace, myDomainCertSecret)) + td.verify(api.upsert("Secret", namespace, myDomainCertSecret, garden.log)) expect(ingresses).to.eql([ { @@ -667,7 +667,10 @@ describe("createIngressResources", () => { err.code = 404 td.when(api.core.readNamespacedSecret("foo", "default")).thenReject(err) - await expectError(async () => await createIngressResources(api, provider, namespace, service), "configuration") + await expectError( + async () => await createIngressResources(api, provider, namespace, service, garden.log), + "configuration" + ) }) it("should throw if a secret for a configured certificate doesn't contain a certificate", async () => { @@ -699,7 +702,10 @@ describe("createIngressResources", () => { err.code = 404 td.when(api.core.readNamespacedSecret("foo", "default")).thenResolve({ data: {} }) - await expectError(async () => await createIngressResources(api, provider, namespace, service), "configuration") + await expectError( + async () => await createIngressResources(api, provider, namespace, service, garden.log), + "configuration" + ) }) it("should throw if a secret for a configured certificate contains an invalid certificate", async () => { @@ -735,7 +741,10 @@ describe("createIngressResources", () => { }, }) - await expectError(async () => await createIngressResources(api, provider, namespace, service), "configuration") + await expectError( + async () => await createIngressResources(api, provider, namespace, service, garden.log), + "configuration" + ) }) it("should correctly match an ingress to a wildcard certificate", async () => { @@ -747,9 +756,9 @@ describe("createIngressResources", () => { }) const api = await getKubeApi(multiTlsProvider) - const ingresses = await createIngressResources(api, multiTlsProvider, namespace, service) + const ingresses = await createIngressResources(api, multiTlsProvider, namespace, service, garden.log) - td.verify(api.upsert("Secret", namespace, wildcardDomainCertSecret)) + td.verify(api.upsert("Secret", namespace, wildcardDomainCertSecret, garden.log)) expect(ingresses).to.eql([ { @@ -818,9 +827,9 @@ describe("createIngressResources", () => { } td.when(api.core.readNamespacedSecret("foo", "default")).thenResolve(myDomainCertSecret) - const ingresses = await createIngressResources(api, provider, namespace, service) + const ingresses = await createIngressResources(api, provider, namespace, service, garden.log) - td.verify(api.upsert("Secret", namespace, myDomainCertSecret)) + td.verify(api.upsert("Secret", namespace, myDomainCertSecret, garden.log)) expect(ingresses).to.eql([ { diff --git a/garden-service/test/unit/src/plugins/kubernetes/container/service.ts b/garden-service/test/unit/src/plugins/kubernetes/container/service.ts index 2408082ec8..a70c8a3bac 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/container/service.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/container/service.ts @@ -12,7 +12,7 @@ describe("createServiceResources", () => { let garden: Garden beforeEach(async () => { - garden = await makeTestGarden(projectRoot, { extraPlugins: [gardenPlugin] }) + garden = await makeTestGarden(projectRoot, { plugins: [gardenPlugin] }) }) it("should return service resources", async () => { diff --git a/package-lock.json b/package-lock.json index 92090e8582..e39d7c1782 100644 --- a/package-lock.json +++ b/package-lock.json @@ -506,6 +506,34 @@ } } }, + "@google-cloud/kms": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@google-cloud/kms/-/kms-1.5.3.tgz", + "integrity": "sha512-egk9hyHLj1w0i7gSXWsUQBZ68xEBk5LSO3KkKgKixoZh9OFboE4Nmh7UY+HBMirIqKznopSfEVOP+Az/ONY4lw==", + "dev": true, + "requires": { + "google-gax": "^1.7.5" + } + }, + "@grpc/grpc-js": { + "version": "0.6.9", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-0.6.9.tgz", + "integrity": "sha512-r1nDOEEiYmAsVYBaS4DPPqdwPOXPw7YhVOnnpPdWhlNtKbYzPash6DqWTTza9gBiYMA5d2Wiq6HzrPqsRaP4yA==", + "dev": true, + "requires": { + "semver": "^6.2.0" + } + }, + "@grpc/proto-loader": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.5.3.tgz", + "integrity": "sha512-8qvUtGg77G2ZT2HqdqYoM/OY97gQd/0crSG34xNmZ4ZOsv3aQT/FQV9QfZPazTGna6MIoyUd+u6AxsoZjJ/VMQ==", + "dev": true, + "requires": { + "lodash.camelcase": "^4.3.0", + "protobufjs": "^6.8.6" + } + }, "@lerna/add": { "version": "3.19.0", "resolved": "https://registry.npmjs.org/@lerna/add/-/add-3.19.0.tgz", @@ -1773,6 +1801,70 @@ "@types/node": ">= 8" } }, + "@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=", + "dev": true + }, + "@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "dev": true + }, + "@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "dev": true + }, + "@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=", + "dev": true + }, + "@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "dev": true, + "requires": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=", + "dev": true + }, + "@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=", + "dev": true + }, + "@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=", + "dev": true + }, + "@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=", + "dev": true + }, + "@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=", + "dev": true + }, "@types/bluebird": { "version": "3.5.29", "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.29.tgz", @@ -1814,12 +1906,34 @@ "@types/node": "*" } }, + "@types/google-cloud__kms": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@types/google-cloud__kms/-/google-cloud__kms-1.5.0.tgz", + "integrity": "sha512-IFQUDHKViQ9RKhmGQFmslpKt6vJzSocZ8u5U4SA6YeQ8fGvoKJ3gkULz63oR+pY5bEvoCReUb8w0blSt47m5IA==", + "dev": true, + "requires": { + "@types/google-protobuf": "*", + "@types/node": "*" + } + }, + "@types/google-protobuf": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/@types/google-protobuf/-/google-protobuf-3.7.2.tgz", + "integrity": "sha512-ifFemzjNchFBCtHS6bZNhSZCBu7tbtOe0e8qY0z2J4HtFXmPJjm6fXSaQsTG7yhShBEZtt2oP/bkwu5k+emlkQ==", + "dev": true + }, "@types/lodash": { "version": "4.14.149", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.149.tgz", "integrity": "sha512-ijGqzZt/b7BfzcK9vTrS6MFljQRPn5BFWOx8oE0GYxribu6uV+aA9zZuXI1zc/etK9E8nrgdoF2+LgUw7+9tJQ==", "dev": true }, + "@types/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz", + "integrity": "sha512-1w52Nyx4Gq47uuu0EVcsHBxZFJgurQ+rTKS3qMHxR1GY2T8c2AJYd6vZoZ9q1rupaDjU0yT+Jc2XTyXkjeMA+Q==", + "dev": true + }, "@types/minimatch": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", @@ -1877,6 +1991,15 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, + "abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, + "requires": { + "event-target-shim": "^5.0.0" + } + }, "agent-base": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", @@ -2379,6 +2502,12 @@ } } }, + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==", + "dev": true + }, "bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -2400,6 +2529,12 @@ "integrity": "sha512-IWIbu7pMqyw3EAJHzzHbWa85b6oud/yfKYg5rqB5hNE8CeMi3nX+2C2sj0HswfblST86hpVEOAb9x34NZd6P7A==", "dev": true }, + "bignumber.js": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", + "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==", + "dev": true + }, "binary-extensions": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz", @@ -2463,6 +2598,12 @@ "integrity": "sha1-WWFrSYME1Var1GaWayLu2j7KX74=", "dev": true }, + "buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=", + "dev": true + }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -4439,6 +4580,15 @@ "safer-buffer": "^2.1.0" } }, + "ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dev": true, + "requires": { + "safe-buffer": "^5.0.1" + } + }, "editorconfig": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.0.tgz", @@ -4664,6 +4814,12 @@ } } }, + "event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dev": true + }, "eventemitter3": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz", @@ -4989,6 +5145,12 @@ "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", "dev": true }, + "fast-text-encoding": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz", + "integrity": "sha512-R9bHCvweUxxwkDwhjav5vxpFvdPGlVngtqmx4pIZfSUhM/Q4NiIUHB456BAf+Q1Nwu3HEZYONtu+Rya+af4jiQ==", + "dev": true + }, "fault": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/fault/-/fault-1.0.3.tgz", @@ -5294,6 +5456,62 @@ "wide-align": "^1.1.0" } }, + "gaxios": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-2.1.0.tgz", + "integrity": "sha512-Gtpb5sdQmb82sgVkT2GnS2n+Kx4dlFwbeMYcDlD395aEvsLCSQXJJcHt7oJ2LrGxDEAeiOkK79Zv2A8Pzt6CFg==", + "dev": true, + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^3.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.3.0" + }, + "dependencies": { + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "https-proxy-agent": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-3.0.1.tgz", + "integrity": "sha512-+ML2Rbh6DAuee7d07tYGEKOEi2voWPUGan+ExdPbPW6Z3svq+JCqr0v8WmKPOkz1vOVykPCBSuobe7G8GJUtVg==", + "dev": true, + "requires": { + "agent-base": "^4.3.0", + "debug": "^3.1.0" + } + }, + "is-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", + "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "gcp-metadata": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-3.2.2.tgz", + "integrity": "sha512-vR7kcJMCYJG/mYWp/a1OszdOqnLB/XW1GorWW1hc1lWVNL26L497zypWb9cG0CYDQ4Bl1Wk0+fSZFFjwJlTQgQ==", + "dev": true, + "requires": { + "gaxios": "^2.1.0", + "json-bigint": "^0.3.0" + } + }, "genfun": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/genfun/-/genfun-5.0.0.tgz", @@ -5802,12 +6020,94 @@ "sparkles": "^1.0.0" } }, + "google-auth-library": { + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-5.5.1.tgz", + "integrity": "sha512-zCtjQccWS/EHYyFdXRbfeSGM/gW+d7uMAcVnvXRnjBXON5ijo6s0nsObP0ifqileIDSbZjTlLtgo+UoN8IFJcg==", + "dev": true, + "requires": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "fast-text-encoding": "^1.0.0", + "gaxios": "^2.1.0", + "gcp-metadata": "^3.2.0", + "gtoken": "^4.1.0", + "jws": "^3.1.5", + "lru-cache": "^5.0.0" + }, + "dependencies": { + "arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "dev": true + }, + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "requires": { + "yallist": "^3.0.2" + } + }, + "yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true + } + } + }, + "google-gax": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-1.11.1.tgz", + "integrity": "sha512-v/APF2G5h2nS5R/1DW2vsgloaMu2/B3xjHdAptR1yUwZpEd9rxPTlhqosrjl/VRu+gWGr9JZN19ZgJTXQ/Db6Q==", + "dev": true, + "requires": { + "@grpc/grpc-js": "0.6.9", + "@grpc/proto-loader": "^0.5.1", + "@types/long": "^4.0.0", + "abort-controller": "^3.0.0", + "duplexify": "^3.6.0", + "google-auth-library": "^5.0.0", + "is-stream-ended": "^0.1.4", + "lodash.at": "^4.6.0", + "lodash.has": "^4.5.2", + "node-fetch": "^2.6.0", + "protobufjs": "^6.8.8", + "retry-request": "^4.0.0", + "semver": "^6.0.0", + "walkdir": "^0.4.0" + } + }, + "google-p12-pem": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-2.0.3.tgz", + "integrity": "sha512-Tq2kBCANxYYPxaBpTgCpRfdoPs9+/lNzc/Iaee4kuMVW5ascD+HwhpBsTLwH85C9Ev4qfB8KKHmpPQYyD2vg2w==", + "dev": true, + "requires": { + "node-forge": "^0.9.0" + } + }, "graceful-fs": { "version": "4.1.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=", "dev": true }, + "gtoken": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-4.1.3.tgz", + "integrity": "sha512-ofW+FiXjswyKdkjMcDbe6E4K7cDDdE82dGDhZIc++kUECqaE7MSErf6arJPAjcnYn1qxE1/Ti06qQuqgVusovQ==", + "dev": true, + "requires": { + "gaxios": "^2.1.0", + "google-p12-pem": "^2.0.0", + "jws": "^3.1.5", + "mime": "^2.2.0" + } + }, "gulp": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/gulp/-/gulp-4.0.2.tgz", @@ -6923,6 +7223,12 @@ "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", "dev": true }, + "is-stream-ended": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-stream-ended/-/is-stream-ended-0.1.4.tgz", + "integrity": "sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==", + "dev": true + }, "is-symbol": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", @@ -7049,6 +7355,15 @@ "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", "dev": true }, + "json-bigint": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz", + "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=", + "dev": true, + "requires": { + "bignumber.js": "^7.0.0" + } + }, "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", @@ -7121,6 +7436,27 @@ "integrity": "sha1-h/zPrv/AtozRnVX2cilD+SnqNeo=", "dev": true }, + "jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "dev": true, + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "dev": true, + "requires": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } + }, "kind-of": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", @@ -7337,6 +7673,18 @@ "integrity": "sha1-+6HEUkwZ7ppfgTa0YJ8BfPTe1pI=", "dev": true }, + "lodash.at": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.at/-/lodash.at-4.6.0.tgz", + "integrity": "sha1-k83OZk8KGZTqM9181A4jr9EbD/g=", + "dev": true + }, + "lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=", + "dev": true + }, "lodash.clonedeep": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", @@ -7364,6 +7712,12 @@ "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=", "dev": true }, + "lodash.has": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz", + "integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI=", + "dev": true + }, "lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", @@ -7436,6 +7790,12 @@ "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=", "dev": true }, + "long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", + "dev": true + }, "longest-streak": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-2.0.3.tgz", @@ -7735,6 +8095,12 @@ "to-regex": "^3.0.2" } }, + "mime": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", + "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==", + "dev": true + }, "mime-db": { "version": "1.40.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", @@ -8032,6 +8398,12 @@ "safe-buffer": "^5.1.1" } }, + "node-forge": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz", + "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ==", + "dev": true + }, "node-gyp": { "version": "5.0.5", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-5.0.5.tgz", @@ -8874,6 +9246,35 @@ "integrity": "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=", "dev": true }, + "protobufjs": { + "version": "6.8.8", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", + "integrity": "sha512-AAmHtD5pXgZfi7GMpllpO3q1Xw1OYldr+dMUlAnffGTAhqkg72WdmSY71uKBF/JuyiKs8psYbtKrhi0ASCD8qw==", + "dev": true, + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.0", + "@types/node": "^10.1.0", + "long": "^4.0.0" + }, + "dependencies": { + "@types/node": { + "version": "10.17.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.6.tgz", + "integrity": "sha512-0a2X6cgN3RdPBL2MIlR6Lt0KlM7fOFsutuXcdglcOq6WvLnYXgPQSh0Mx6tO1KCAE8MxbHSOSTWDoUxRq+l3DA==", + "dev": true + } + } + }, "protocols": { "version": "1.4.7", "resolved": "https://registry.npmjs.org/protocols/-/protocols-1.4.7.tgz", @@ -9603,6 +10004,42 @@ "integrity": "sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q=", "dev": true }, + "retry-request": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz", + "integrity": "sha512-BINDzVtLI2BDukjWmjAIRZ0oglnCAkpP2vQjM3jdLhmT62h0xnQgciPwBRDAvHqpkPT2Wo1XuUyLyn6nbGrZQQ==", + "dev": true, + "requires": { + "debug": "^4.1.1", + "through2": "^3.0.1" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "through2": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz", + "integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==", + "dev": true, + "requires": { + "readable-stream": "2 || 3" + } + } + } + }, "rimraf": { "version": "2.6.3", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", @@ -11430,6 +11867,12 @@ "vinyl": "^2.0.0" } }, + "walkdir": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.4.1.tgz", + "integrity": "sha512-3eBwRyEln6E1MSzcxcVpQIhRG8Q1jLvEqRmCZqS3dsfXEDR/AhOF4d+jHg1qvDCpYaVRZjENPQyrVxAkQqxPgQ==", + "dev": true + }, "wcwidth": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", diff --git a/package.json b/package.json index a13902598b..b33207ed64 100644 --- a/package.json +++ b/package.json @@ -14,9 +14,11 @@ "devDependencies": { "@commitlint/cli": "^8.2.0", "@commitlint/config-conventional": "^8.2.0", + "@google-cloud/kms": "^1.5.3", "@lerna/version": "^3.18.5", "@types/bluebird": "^3.5.29", "@types/fs-extra": "^8.0.1", + "@types/google-cloud__kms": "^1.5.0", "@types/lodash": "^4.14.149", "@types/node": "^12.12.14", "@types/semver": "^6.2.0", diff --git a/secrets/README.md b/secrets/README.md new file mode 100644 index 0000000000..4ff2bb53f4 --- /dev/null +++ b/secrets/README.md @@ -0,0 +1,5 @@ +# Secrets + +Files in this directory are created using the `bin/encrypt-file.ts` helper, and can only be decrypted using a Google Cloud KMS key, which is made available to Garden team developers. + +We don't store particularly sensitive data here, i.e. nothing that exposes sensitive data or systems, but regardless we want to keep these secure. These files are generally only used for testing purposes. diff --git a/secrets/test-docker-auth.json b/secrets/test-docker-auth.json new file mode 100644 index 0000000000000000000000000000000000000000..a76651cc3ac1118a254e619b4bc3e7a272baed30 GIT binary patch literal 586 zcmV-Q0=4}LBmmi!Wt6SOvT=)w9MqPuSE)+`kQS>CxaI7u+StX3Y(lOPNfMz10DBgp zzRG;r_g^_Lsd93~jx$`{%Z9S`0!p!ooWQK<5D;N`*VD{@fd6mpVePgOYDD^*dPtS$-Fr2i1fUDc-3Lxo-9pS zm+b`Dt3R=K?0&EDx3w@(;{gC>_JMH;-(c?`M~7h0WGR01!2Ikl;QYymeFON>dcIXS zMjAu?9WFgTOpEtHV<_sO-y>T1JPLxLB{apB6hYoqp`3|H5;uHNWyLvZLrSqK*-7B7 zw|fFzwS1PT^m<)NZqxPBBbgJ?-@0^yrRZ=$gh|-|RI8{q7{uSo9os0u_A4LC(d8!h zn+}NOAH_myuMjo;n#D9zxv3iB2CDs>83&U{^el>(&Why$iHWI!{Y%fqi6pv%2@4uq zVYML4Cbb(ZZ7ZVL`AVB@&6x)X*XFLkRTQ*zO&%!=C;(LZcvogL<3M}GCeSspunbkg Yf@!nVtVzZV{!H(&fq(xwboW;Sb>{*cZ~y=R literal 0 HcmV?d00001