From 88f5b8e33103f0997a947e0c301b3f2c67f41c21 Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Fri, 31 May 2019 15:26:45 +0200 Subject: [PATCH] feat(k8s): in-cluster building Moving this from our closed-source version. This adds an option to build container images inside Kubernetes clusters, and by extension removes the need for Docker and Kubernetes running on developer machines. Use it by setting `buildMode: "cluster-docker"` in the `kubernetes` provider configuration. --- docs/reference/providers/kubernetes.md | 210 ++++++++++++- docs/reference/providers/local-kubernetes.md | 210 ++++++++++++- examples/simple-project/garden.yml | 8 + examples/vote-helm/api/garden.yml | 2 +- examples/vote-helm/result/garden.yml | 2 +- examples/vote-helm/vote/garden.yml | 1 + examples/vote-helm/worker/garden.yml | 1 + examples/vote/garden.yml | 8 + garden-service/bin/push-containers.sh | 20 ++ garden-service/gcloud.Dockerfile | 10 + garden-service/package-lock.json | 42 +-- garden-service/src/docs/config.ts | 2 +- .../src/plugins/container/container.ts | 2 +- .../src/plugins/kubernetes/config.ts | 277 ++++++++++++++++++ .../src/plugins/kubernetes/container/build.ts | 180 +++++++++++- .../kubernetes/container/deployment.ts | 2 +- .../plugins/kubernetes/container/handlers.ts | 8 +- .../plugins/kubernetes/container/ingress.ts | 2 +- .../src/plugins/kubernetes/container/logs.ts | 2 +- .../src/plugins/kubernetes/container/run.ts | 2 +- .../plugins/kubernetes/container/status.ts | 2 +- .../src/plugins/kubernetes/helm/build.ts | 2 +- .../src/plugins/kubernetes/helm/common.ts | 2 +- .../src/plugins/kubernetes/helm/deployment.ts | 2 +- .../src/plugins/kubernetes/helm/hot-reload.ts | 2 +- .../src/plugins/kubernetes/helm/logs.ts | 2 +- .../src/plugins/kubernetes/helm/run.ts | 2 +- .../src/plugins/kubernetes/helm/status.ts | 2 +- .../src/plugins/kubernetes/helm/test.ts | 2 +- .../src/plugins/kubernetes/helm/tiller.ts | 2 +- .../src/plugins/kubernetes/hot-reload.ts | 2 +- garden-service/src/plugins/kubernetes/init.ts | 44 ++- .../kubernetes/kubernetes-module/handlers.ts | 2 +- .../src/plugins/kubernetes/kubernetes.ts | 155 ++-------- .../src/plugins/kubernetes/local/config.ts | 5 +- .../src/plugins/kubernetes/namespace.ts | 2 +- .../src/plugins/kubernetes/secrets.ts | 2 +- .../src/plugins/kubernetes/status.ts | 2 +- .../src/plugins/kubernetes/system.ts | 2 +- .../src/plugins/kubernetes/task-results.ts | 2 +- garden-service/src/plugins/kubernetes/test.ts | 2 +- garden-service/src/plugins/kubernetes/util.ts | 2 +- .../src/plugins/openfaas/openfaas.ts | 2 +- .../system/docker-daemon/Chart.yaml | 5 + .../system/docker-daemon/garden.yml | 23 ++ .../system/docker-daemon/templates/NOTES.txt | 15 + .../docker-daemon/templates/_helpers.tpl | 32 ++ .../docker-daemon/templates/deployment.yaml | 115 ++++++++ .../docker-daemon/templates/service.yaml | 19 ++ .../docker-daemon/templates/volume.yaml | 35 +++ .../system/docker-daemon/values.yaml | 45 +++ .../system/docker-registry/garden.yml | 23 ++ .../system/registry-proxy/Chart.yaml | 5 + .../system/registry-proxy/garden.yml | 10 + .../registry-proxy/templates/_helpers.tpl | 32 ++ .../registry-proxy/templates/daemonset.yaml | 78 +++++ .../system/registry-proxy/values.yaml | 22 ++ .../plugins/kubernetes/container/ingress.ts | 10 +- 58 files changed, 1496 insertions(+), 208 deletions(-) create mode 100755 garden-service/bin/push-containers.sh create mode 100644 garden-service/gcloud.Dockerfile create mode 100644 garden-service/src/plugins/kubernetes/config.ts create mode 100644 garden-service/static/kubernetes/system/docker-daemon/Chart.yaml create mode 100644 garden-service/static/kubernetes/system/docker-daemon/garden.yml create mode 100644 garden-service/static/kubernetes/system/docker-daemon/templates/NOTES.txt create mode 100644 garden-service/static/kubernetes/system/docker-daemon/templates/_helpers.tpl create mode 100644 garden-service/static/kubernetes/system/docker-daemon/templates/deployment.yaml create mode 100644 garden-service/static/kubernetes/system/docker-daemon/templates/service.yaml create mode 100644 garden-service/static/kubernetes/system/docker-daemon/templates/volume.yaml create mode 100644 garden-service/static/kubernetes/system/docker-daemon/values.yaml create mode 100644 garden-service/static/kubernetes/system/docker-registry/garden.yml create mode 100644 garden-service/static/kubernetes/system/registry-proxy/Chart.yaml create mode 100644 garden-service/static/kubernetes/system/registry-proxy/garden.yml create mode 100644 garden-service/static/kubernetes/system/registry-proxy/templates/_helpers.tpl create mode 100644 garden-service/static/kubernetes/system/registry-proxy/templates/daemonset.yaml create mode 100644 garden-service/static/kubernetes/system/registry-proxy/values.yaml diff --git a/docs/reference/providers/kubernetes.md b/docs/reference/providers/kubernetes.md index 9b6a783365e..39b7f7974bd 100644 --- a/docs/reference/providers/kubernetes.md +++ b/docs/reference/providers/kubernetes.md @@ -131,6 +131,15 @@ sources: | Type | Required | | ---- | -------- | | `array[object]` | No +### `environments[].providers[].buildMode` +[environments](#environments) > [providers](#environments[].providers[]) > buildMode + +Choose the mechanism used to build containers before deploying. By default it uses the local docker, but you can set it to 'cluster-docker' to sync files to a remote docker daemon, installed in the cluster, and build container images there. +This is currently experimental and sometimes not needed (e.g. with Docker for Desktop), so it's not enabled by default. + +| Type | Required | +| ---- | -------- | +| `string` | No ### `environments[].providers[].defaultHostname` [environments](#environments) > [providers](#environments[].providers[]) > defaultHostname @@ -196,6 +205,182 @@ The namespace where the secret is stored. If necessary, the secret may be copied | Type | Required | | ---- | -------- | | `string` | No +### `environments[].providers[].storage` +[environments](#environments) > [providers](#environments[].providers[]) > storage + +Storage parameters to set for the in-cluster builder and container registry persistent volume (which are automatically installed and used when buildMode=cluster-docker). + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].storage.builder` +[environments](#environments) > [providers](#environments[].providers[]) > [storage](#environments[].providers[].storage) > builder + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].storage.builder.size` +[environments](#environments) > [providers](#environments[].providers[]) > [storage](#environments[].providers[].storage) > [builder](#environments[].providers[].storage.builder) > size + +Volume size for the registry in megabytes. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].storage.builder.storageClass` +[environments](#environments) > [providers](#environments[].providers[]) > [storage](#environments[].providers[].storage) > [builder](#environments[].providers[].storage.builder) > storageClass + +Storage class to use for the volume. + +| Type | Required | +| ---- | -------- | +| `string` | No +### `environments[].providers[].storage.registry` +[environments](#environments) > [providers](#environments[].providers[]) > [storage](#environments[].providers[].storage) > registry + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].storage.registry.size` +[environments](#environments) > [providers](#environments[].providers[]) > [storage](#environments[].providers[].storage) > [registry](#environments[].providers[].storage.registry) > size + +Volume size for the registry in megabytes. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].storage.registry.storageClass` +[environments](#environments) > [providers](#environments[].providers[]) > [storage](#environments[].providers[].storage) > [registry](#environments[].providers[].storage.registry) > storageClass + +Storage class to use for the volume. + +| Type | Required | +| ---- | -------- | +| `string` | No +### `environments[].providers[].resources` +[environments](#environments) > [providers](#environments[].providers[]) > resources + +Resource requests and limits for the in-cluster builder and container registry (which are automatically installed and used when buildMode=cluster-docker). + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.builder` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > builder + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.builder.limits` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [builder](#environments[].providers[].resources.builder) > limits + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.builder.limits.cpu` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [builder](#environments[].providers[].resources.builder) > [limits](#environments[].providers[].resources.builder.limits) > cpu + +CPU limit in millicpu. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.builder.limits.memory` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [builder](#environments[].providers[].resources.builder) > [limits](#environments[].providers[].resources.builder.limits) > memory + +Memory limit in megabytes. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.builder.requests` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [builder](#environments[].providers[].resources.builder) > requests + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.builder.requests.cpu` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [builder](#environments[].providers[].resources.builder) > [requests](#environments[].providers[].resources.builder.requests) > cpu + +CPU request in millicpu. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.builder.requests.memory` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [builder](#environments[].providers[].resources.builder) > [requests](#environments[].providers[].resources.builder.requests) > memory + +Memory request in megabytes. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.registry` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > registry + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.registry.limits` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [registry](#environments[].providers[].resources.registry) > limits + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.registry.limits.cpu` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [registry](#environments[].providers[].resources.registry) > [limits](#environments[].providers[].resources.registry.limits) > cpu + +CPU limit in millicpu. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.registry.limits.memory` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [registry](#environments[].providers[].resources.registry) > [limits](#environments[].providers[].resources.registry.limits) > memory + +Memory limit in megabytes. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.registry.requests` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [registry](#environments[].providers[].resources.registry) > requests + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.registry.requests.cpu` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [registry](#environments[].providers[].resources.registry) > [requests](#environments[].providers[].resources.registry.requests) > cpu + +CPU request in millicpu. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.registry.requests.memory` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [registry](#environments[].providers[].resources.registry) > [requests](#environments[].providers[].resources.registry.requests) > memory + +Memory request in megabytes. + +| Type | Required | +| ---- | -------- | +| `number` | No ### `environments[].providers[].tlsCertificates[]` [environments](#environments) > [providers](#environments[].providers[]) > tlsCertificates @@ -413,12 +598,35 @@ sources: repositoryUrl: environments: - providers: - - defaultHostname: + - buildMode: local + defaultHostname: defaultUsername: forceSsl: false imagePullSecrets: - name: namespace: default + storage: + builder: + size: 10240 + storageClass: '-' + registry: + size: 10240 + storageClass: '-' + resources: + builder: + limits: + cpu: 2000 + memory: 4096 + requests: + cpu: 200 + memory: 512 + registry: + limits: + cpu: 2000 + memory: 4096 + requests: + cpu: 200 + memory: 512 tlsCertificates: - name: hostnames: diff --git a/docs/reference/providers/local-kubernetes.md b/docs/reference/providers/local-kubernetes.md index 63f53a1d90e..1df11cc32dd 100644 --- a/docs/reference/providers/local-kubernetes.md +++ b/docs/reference/providers/local-kubernetes.md @@ -131,6 +131,15 @@ sources: | Type | Required | | ---- | -------- | | `array[object]` | No +### `environments[].providers[].buildMode` +[environments](#environments) > [providers](#environments[].providers[]) > buildMode + +Choose the mechanism used to build containers before deploying. By default it uses the local docker, but you can set it to 'cluster-docker' to sync files to a remote docker daemon, installed in the cluster, and build container images there. +This is currently experimental and sometimes not needed (e.g. with Docker for Desktop), so it's not enabled by default. + +| Type | Required | +| ---- | -------- | +| `string` | No ### `environments[].providers[].defaultHostname` [environments](#environments) > [providers](#environments[].providers[]) > defaultHostname @@ -196,6 +205,182 @@ The namespace where the secret is stored. If necessary, the secret may be copied | Type | Required | | ---- | -------- | | `string` | No +### `environments[].providers[].storage` +[environments](#environments) > [providers](#environments[].providers[]) > storage + +Storage parameters to set for the in-cluster builder and container registry persistent volume (which are automatically installed and used when buildMode=cluster-docker). + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].storage.builder` +[environments](#environments) > [providers](#environments[].providers[]) > [storage](#environments[].providers[].storage) > builder + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].storage.builder.size` +[environments](#environments) > [providers](#environments[].providers[]) > [storage](#environments[].providers[].storage) > [builder](#environments[].providers[].storage.builder) > size + +Volume size for the registry in megabytes. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].storage.builder.storageClass` +[environments](#environments) > [providers](#environments[].providers[]) > [storage](#environments[].providers[].storage) > [builder](#environments[].providers[].storage.builder) > storageClass + +Storage class to use for the volume. + +| Type | Required | +| ---- | -------- | +| `string` | No +### `environments[].providers[].storage.registry` +[environments](#environments) > [providers](#environments[].providers[]) > [storage](#environments[].providers[].storage) > registry + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].storage.registry.size` +[environments](#environments) > [providers](#environments[].providers[]) > [storage](#environments[].providers[].storage) > [registry](#environments[].providers[].storage.registry) > size + +Volume size for the registry in megabytes. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].storage.registry.storageClass` +[environments](#environments) > [providers](#environments[].providers[]) > [storage](#environments[].providers[].storage) > [registry](#environments[].providers[].storage.registry) > storageClass + +Storage class to use for the volume. + +| Type | Required | +| ---- | -------- | +| `string` | No +### `environments[].providers[].resources` +[environments](#environments) > [providers](#environments[].providers[]) > resources + +Resource requests and limits for the in-cluster builder and container registry (which are automatically installed and used when buildMode=cluster-docker). + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.builder` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > builder + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.builder.limits` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [builder](#environments[].providers[].resources.builder) > limits + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.builder.limits.cpu` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [builder](#environments[].providers[].resources.builder) > [limits](#environments[].providers[].resources.builder.limits) > cpu + +CPU limit in millicpu. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.builder.limits.memory` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [builder](#environments[].providers[].resources.builder) > [limits](#environments[].providers[].resources.builder.limits) > memory + +Memory limit in megabytes. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.builder.requests` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [builder](#environments[].providers[].resources.builder) > requests + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.builder.requests.cpu` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [builder](#environments[].providers[].resources.builder) > [requests](#environments[].providers[].resources.builder.requests) > cpu + +CPU request in millicpu. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.builder.requests.memory` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [builder](#environments[].providers[].resources.builder) > [requests](#environments[].providers[].resources.builder.requests) > memory + +Memory request in megabytes. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.registry` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > registry + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.registry.limits` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [registry](#environments[].providers[].resources.registry) > limits + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.registry.limits.cpu` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [registry](#environments[].providers[].resources.registry) > [limits](#environments[].providers[].resources.registry.limits) > cpu + +CPU limit in millicpu. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.registry.limits.memory` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [registry](#environments[].providers[].resources.registry) > [limits](#environments[].providers[].resources.registry.limits) > memory + +Memory limit in megabytes. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.registry.requests` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [registry](#environments[].providers[].resources.registry) > requests + + + +| Type | Required | +| ---- | -------- | +| `object` | No +### `environments[].providers[].resources.registry.requests.cpu` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [registry](#environments[].providers[].resources.registry) > [requests](#environments[].providers[].resources.registry.requests) > cpu + +CPU request in millicpu. + +| Type | Required | +| ---- | -------- | +| `number` | No +### `environments[].providers[].resources.registry.requests.memory` +[environments](#environments) > [providers](#environments[].providers[]) > [resources](#environments[].providers[].resources) > [registry](#environments[].providers[].resources.registry) > [requests](#environments[].providers[].resources.registry.requests) > memory + +Memory request in megabytes. + +| Type | Required | +| ---- | -------- | +| `number` | No ### `environments[].providers[].tlsCertificates[]` [environments](#environments) > [providers](#environments[].providers[]) > tlsCertificates @@ -346,12 +531,35 @@ sources: repositoryUrl: environments: - providers: - - defaultHostname: + - buildMode: local + defaultHostname: defaultUsername: forceSsl: false imagePullSecrets: - name: namespace: default + storage: + builder: + size: 10240 + storageClass: '-' + registry: + size: 10240 + storageClass: '-' + resources: + builder: + limits: + cpu: 2000 + memory: 4096 + requests: + cpu: 200 + memory: 512 + registry: + limits: + cpu: 2000 + memory: 4096 + requests: + cpu: 200 + memory: 512 tlsCertificates: - name: hostnames: diff --git a/examples/simple-project/garden.yml b/examples/simple-project/garden.yml index 00f2e5a4ab3..7e718c487d3 100644 --- a/examples/simple-project/garden.yml +++ b/examples/simple-project/garden.yml @@ -4,3 +4,11 @@ project: - name: local providers: - name: local-kubernetes + - name: remote + providers: + - name: kubernetes + # Replace these values as appropriate + context: gke_garden-dev-200012_europe-west1-b_garden-dev-1 + namespace: ${local.env.USER || "default"}-simple-project + defaultHostname: ${local.env.USER || "default"}-simple-project.dev-1.sys.garden + buildMode: cluster-docker diff --git a/examples/vote-helm/api/garden.yml b/examples/vote-helm/api/garden.yml index 22bace07f0a..26ea10d9940 100644 --- a/examples/vote-helm/api/garden.yml +++ b/examples/vote-helm/api/garden.yml @@ -10,7 +10,7 @@ module: values: name: api image: - repository: api-image + repository: ${modules.api-image.outputs.deployment-image-name} tag: ${modules.api-image.version} ingress: enabled: true diff --git a/examples/vote-helm/result/garden.yml b/examples/vote-helm/result/garden.yml index 7e77eb0f7f8..9ea041bdc3e 100644 --- a/examples/vote-helm/result/garden.yml +++ b/examples/vote-helm/result/garden.yml @@ -11,7 +11,7 @@ module: values: name: result image: - repository: result-image + repository: ${modules.result-image.outputs.deployment-image-name} tag: ${modules.result-image.version} ingress: enabled: true diff --git a/examples/vote-helm/vote/garden.yml b/examples/vote-helm/vote/garden.yml index 7e1c187ddae..32e30e09815 100644 --- a/examples/vote-helm/vote/garden.yml +++ b/examples/vote-helm/vote/garden.yml @@ -9,6 +9,7 @@ module: - api values: image: + repository: ${modules.vote-image.outputs.deployment-image-name} tag: ${modules.vote-image.version} ingress: enabled: true diff --git a/examples/vote-helm/worker/garden.yml b/examples/vote-helm/worker/garden.yml index 788504c2ce8..be061085335 100644 --- a/examples/vote-helm/worker/garden.yml +++ b/examples/vote-helm/worker/garden.yml @@ -8,4 +8,5 @@ module: dependencies: [worker-image] values: image: + repository: ${modules.worker-image.outputs.deployment-image-name} tag: ${modules.worker-image.version} diff --git a/examples/vote/garden.yml b/examples/vote/garden.yml index ce26b464974..b0083e4ddb0 100644 --- a/examples/vote/garden.yml +++ b/examples/vote/garden.yml @@ -4,3 +4,11 @@ project: - name: local providers: - name: local-kubernetes + - name: remote + providers: + - name: kubernetes + # Replace these values as appropriate + context: gke_garden-dev-200012_europe-west1-b_garden-dev-1 + namespace: ${local.env.USER || "default"}-vote + defaultHostname: ${local.env.USER || "default"}-vote.dev-1.sys.garden + buildMode: cluster-docker diff --git a/garden-service/bin/push-containers.sh b/garden-service/bin/push-containers.sh new file mode 100755 index 00000000000..dcc519fd572 --- /dev/null +++ b/garden-service/bin/push-containers.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e + +garden_service_root=$(cd `dirname $0` && cd .. && pwd) + +cd ${garden_service_root} + +args=( $@ ) +version=${args[0]:-$(git rev-parse --short HEAD)} + +echo "Building version ${version}" + +npm run build + +docker build -t gardendev/garden:${version} . +docker build -t gardendev/garden-gcloud:${version} --build-arg NAME=garden --build-arg VERSION=${version} -f gcloud.Dockerfile . + +echo "Pushing images" + +docker push gardendev/garden:${version} +docker push gardendev/garden-gcloud:${version} diff --git a/garden-service/gcloud.Dockerfile b/garden-service/gcloud.Dockerfile new file mode 100644 index 00000000000..69fd2707445 --- /dev/null +++ b/garden-service/gcloud.Dockerfile @@ -0,0 +1,10 @@ +ARG NAME +ARG VERSION +FROM gardendev/${NAME:-garden}:${VERSION} + +RUN apk add --no-cache python \ + && mkdir -p /gcloud \ + && curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz | tar xz -C /gcloud \ + && /gcloud/google-cloud-sdk/install.sh --quiet \ + && ln -s /gcloud/google-cloud-sdk/bin/gcloud /usr/local/bin/gcloud \ + && chmod +x /usr/local/bin/gcloud diff --git a/garden-service/package-lock.json b/garden-service/package-lock.json index fe2946efab0..57a2638048d 100644 --- a/garden-service/package-lock.json +++ b/garden-service/package-lock.json @@ -157,7 +157,7 @@ "dependencies": { "chalk": { "version": "2.3.1", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.1.tgz", + "resolved": "http://registry.npmjs.org/chalk/-/chalk-2.3.1.tgz", "integrity": "sha512-QUU4ofkDoMIVO7hcx1iPTISs88wsO8jA92RQIm4JAwZvFGGAV2hSAA1NX7oVj2Ej2Q6NDTcRDjPTFrMCRZoJ6g==", "dev": true, "requires": { @@ -3262,7 +3262,7 @@ }, "minimist": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", "dev": true } @@ -4421,7 +4421,7 @@ }, "es6-promisify": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", + "resolved": "http://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", "dev": true, "requires": { @@ -5102,7 +5102,7 @@ }, "readable-stream": { "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", "dev": true, "requires": { @@ -5245,7 +5245,7 @@ }, "minimist": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", "dev": true } @@ -6105,7 +6105,7 @@ }, "got": { "version": "6.7.1", - "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", + "resolved": "http://registry.npmjs.org/got/-/got-6.7.1.tgz", "integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=", "dev": true, "requires": { @@ -6893,7 +6893,7 @@ }, "http-errors": { "version": "1.6.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", "requires": { "depd": "~1.1.2", @@ -7268,7 +7268,7 @@ }, "is-obj": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=", "dev": true }, @@ -7469,7 +7469,7 @@ }, "async": { "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" }, "escodegen": { @@ -9233,7 +9233,7 @@ }, "async": { "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true }, @@ -9280,7 +9280,7 @@ }, "os-locale": { "version": "1.4.0", - "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "resolved": "http://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", "dev": true, "requires": { @@ -9300,7 +9300,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "dev": true, "requires": { @@ -9309,7 +9309,7 @@ }, "wrap-ansi": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", "dev": true, "requires": { @@ -9325,7 +9325,7 @@ }, "yargs": { "version": "3.32.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.32.0.tgz", + "resolved": "http://registry.npmjs.org/yargs/-/yargs-3.32.0.tgz", "integrity": "sha1-AwiOnr+edWtpdRYR0qXvWRSCyZU=", "dev": true, "requires": { @@ -10411,7 +10411,7 @@ "dependencies": { "minimist": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" } } @@ -10616,7 +10616,7 @@ }, "kind-of": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/kind-of/-/kind-of-1.1.0.tgz", "integrity": "sha1-FAo9LUGjbS78+pN3tiwk+ElaXEQ=" }, "plugin-error": { @@ -10638,7 +10638,7 @@ }, "readable-stream": { "version": "2.0.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=", "requires": { "core-util-is": "~1.0.0", @@ -10656,12 +10656,12 @@ }, "string_decoder": { "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" }, "through2": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.1.tgz", + "resolved": "http://registry.npmjs.org/through2/-/through2-2.0.1.tgz", "integrity": "sha1-OE51MU1J8y3hLuu4E2uOtrXVnak=", "requires": { "readable-stream": "~2.0.0", @@ -11003,7 +11003,7 @@ "dependencies": { "kind-of": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz", + "resolved": "http://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz", "integrity": "sha1-AY7HpM5+OobLkUG+UZ0kyPqpgbU=", "dev": true, "requires": { @@ -13132,7 +13132,7 @@ }, "xmlbuilder": { "version": "9.0.7", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "resolved": "http://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", "dev": true }, diff --git a/garden-service/src/docs/config.ts b/garden-service/src/docs/config.ts index 31fa22342a2..5f7ca6a1c35 100644 --- a/garden-service/src/docs/config.ts +++ b/garden-service/src/docs/config.ts @@ -25,7 +25,7 @@ import { projectSchema } from "../config/project" import { baseModuleSpecSchema } from "../config/module" import handlebars = require("handlebars") import { configSchema as localK8sConfigSchema } from "../plugins/kubernetes/local/config" -import { configSchema as k8sConfigSchema } from "../plugins/kubernetes/kubernetes" +import { configSchema as k8sConfigSchema } from "../plugins/kubernetes/config" import { configSchema as openfaasConfigSchema } from "../plugins/openfaas/openfaas" import { joiArray } from "../config/common" import { mavenContainerConfigSchema } from "../plugins/maven-container/maven-container" diff --git a/garden-service/src/plugins/container/container.ts b/garden-service/src/plugins/container/container.ts index 5d997b16691..f6e090dc789 100644 --- a/garden-service/src/plugins/container/container.ts +++ b/garden-service/src/plugins/container/container.ts @@ -14,7 +14,7 @@ import { GardenPlugin } from "../../types/plugin/plugin" import { containerHelpers } from "./helpers" import { ContainerModule, containerModuleSpecSchema } from "./config" import { buildContainerModule, getContainerBuildStatus } from "./build" -import { KubernetesProvider } from "../kubernetes/kubernetes" +import { KubernetesProvider } from "../kubernetes/config" import { ConfigureModuleParams } from "../../types/plugin/module/configure" import { PublishModuleParams } from "../../types/plugin/module/publishModule" import { HotReloadServiceParams } from "../../types/plugin/service/hotReloadService" diff --git a/garden-service/src/plugins/kubernetes/config.ts b/garden-service/src/plugins/kubernetes/config.ts new file mode 100644 index 00000000000..39d8e31fe44 --- /dev/null +++ b/garden-service/src/plugins/kubernetes/config.ts @@ -0,0 +1,277 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import * as Joi from "joi" +import dedent = require("dedent") + +import { joiArray, joiIdentifier, joiProviderName } from "../../config/common" +import { Provider, providerConfigBaseSchema, ProviderConfig } from "../../config/project" +import { containerRegistryConfigSchema, ContainerRegistryConfig } from "../container/config" +import { PluginContext } from "../../plugin-context" +import { deline } from "../../util/string" + +export const name = "kubernetes" + +export interface SecretRef { + name: string + namespace: string +} + +export interface IngressTlsCertificate { + name: string + hostnames?: string[] + secretRef: SecretRef +} + +interface KubernetesResourceSpec { + limits: { + cpu: number, + memory: number, + }, + requests: { + cpu: number, + memory: number, + } +} + +interface KubernetesResources { + builder: KubernetesResourceSpec + registry: KubernetesResourceSpec +} + +interface KubernetesStorageSpec { + size: number + storageClass: string +} + +interface KubernetesStorage { + builder: KubernetesStorageSpec + registry: KubernetesStorageSpec +} + +export interface KubernetesBaseConfig extends ProviderConfig { + buildMode: string + context: string + defaultHostname?: string + defaultUsername?: string + forceSsl: boolean + imagePullSecrets: SecretRef[] + ingressHttpPort: number + ingressHttpsPort: number + ingressClass?: string + namespace?: string + resources: KubernetesResources + storage: KubernetesStorage + tlsCertificates: IngressTlsCertificate[] + _systemServices: string[] +} + +export interface KubernetesConfig extends KubernetesBaseConfig { + deploymentRegistry?: ContainerRegistryConfig +} + +export type KubernetesProvider = Provider +export type KubernetesPluginContext = PluginContext + +export const defaultResources: KubernetesResources = { + builder: { + limits: { + cpu: 2000, + memory: 4096, + }, + requests: { + cpu: 200, + memory: 512, + }, + }, + registry: { + limits: { + cpu: 2000, + memory: 4096, + }, + requests: { + cpu: 200, + memory: 512, + }, + }, +} + +export const defaultStorage: KubernetesStorage = { + builder: { + size: 10 * 1024, + storageClass: "-", + }, + registry: { + size: 10 * 1024, + storageClass: "-", + }, +} + +const resourceSchema = (defaults: KubernetesResourceSpec) => Joi.object() + .keys({ + limits: Joi.object() + .keys({ + cpu: Joi.number() + .integer() + .default(defaults.limits.cpu) + .description("CPU limit in millicpu."), + memory: Joi.number() + .integer() + .default(defaults.limits.memory) + .description("Memory limit in megabytes."), + }) + .default(defaults.limits), + requests: Joi.object() + .keys({ + cpu: Joi.number() + .integer() + .default(defaults.requests.cpu) + .description("CPU request in millicpu."), + memory: Joi.number() + .integer() + .default(defaults.requests.memory) + .description("Memory request in megabytes."), + }) + .default(defaults.requests), + }) + .default(defaults) + +const storageSchema = (defaults: KubernetesStorageSpec) => Joi.object() + .keys({ + size: Joi.number() + .integer() + .default(defaults.size) + .description("Volume size for the registry in megabytes."), + storageClass: Joi.string() + .default(defaults.storageClass) + .description("Storage class to use for the volume."), + }) + .default(defaults) + +export const k8sContextSchema = Joi.string() + .required() + .description("The kubectl context to use to connect to the Kubernetes cluster.") + .example("my-dev-context") + +const secretRef = Joi.object() + .keys({ + name: joiIdentifier() + .required() + .description("The name of the Kubernetes secret.") + .example("my-secret"), + namespace: joiIdentifier() + .default("default") + .description( + "The namespace where the secret is stored. " + + "If necessary, the secret may be copied to the appropriate namespace before use.", + ), + }) + .description("Reference to a Kubernetes secret.") + +const imagePullSecretsSchema = joiArray(secretRef) + .description(dedent` + References to \`docker-registry\` secrets to use for authenticating with remote registries when pulling + images. This is necessary if you reference private images in your module configuration, and is required + when configuring a remote Kubernetes environment. + `) + +const tlsCertificateSchema = Joi.object() + .keys({ + name: joiIdentifier() + .required() + .description("A unique identifier for this certificate.") + .example("www") + .example("wildcard"), + hostnames: Joi.array().items(Joi.string().hostname()) + .description( + "A list of hostnames that this certificate should be used for. " + + "If you don't specify these, they will be automatically read from the certificate.", + ) + .example([["www.mydomain.com"], {}]), + secretRef: secretRef + .description("A reference to the Kubernetes secret that contains the TLS certificate and key for the domain.") + .example({ name: "my-tls-secret", namespace: "default" }), + }) + +export const kubernetesConfigBase = providerConfigBaseSchema + .keys({ + buildMode: Joi.string() + .allow("local", "cluster-docker") + .default("local") + .description(deline` + Choose the mechanism used to build containers before deploying. By default it uses the local docker, but you + can set it to 'cluster-docker' to sync files to a remote docker daemon, installed in the cluster, and build + container images there. + + This is currently experimental and sometimes not needed (e.g. with Docker for Desktop), so it's not enabled + by default. + `), + defaultHostname: Joi.string() + .description("A default hostname to use when no hostname is explicitly configured for a service.") + .example("api.mydomain.com"), + defaultUsername: joiIdentifier() + .description("Set a default username (used for namespacing within a cluster)."), + forceSsl: Joi.boolean() + .default(false) + .description( + "Require SSL on all services. If set to true, an error is raised when no certificate " + + "is available for a configured hostname.", + ), + imagePullSecrets: imagePullSecretsSchema, + storage: Joi.object() + .keys({ + builder: storageSchema(defaultStorage.builder), + registry: storageSchema(defaultStorage.registry), + }) + .default(defaultStorage) + .description(deline` + Storage parameters to set for the in-cluster builder and container registry persistent volume + (which are automatically installed and used when buildMode=cluster-docker). + `), + resources: Joi.object() + .keys({ + builder: resourceSchema(defaultResources.builder), + registry: resourceSchema(defaultResources.registry), + }) + .default(defaultResources) + .description(deline` + Resource requests and limits for the in-cluster builder and container registry + (which are automatically installed and used when buildMode=cluster-docker). + `), + tlsCertificates: joiArray(tlsCertificateSchema) + .unique("name") + .description("One or more certificates to use for ingress."), + _systemServices: joiArray(joiIdentifier()) + .meta({ internal: true }), + }) + +export const configSchema = kubernetesConfigBase + .keys({ + name: joiProviderName("kubernetes"), + context: k8sContextSchema + .required(), + deploymentRegistry: containerRegistryConfigSchema, + ingressClass: Joi.string() + .description(dedent` + The ingress class to use on configured Ingresses (via the \`kubernetes.io/ingress.class\` annotation) + when deploying \`container\` services. Use this if you have multiple ingress controllers in your cluster. + `), + ingressHttpPort: Joi.number() + .default(80) + .description("The external HTTP port of the cluster's ingress controller."), + ingressHttpsPort: Joi.number() + .default(443) + .description("The external HTTPS port of the cluster's ingress controller."), + namespace: Joi.string() + .default(undefined, "") + .description( + "Specify which namespace to deploy services to (defaults to ). " + + "Note that the framework generates other namespaces as well with this name as a prefix.", + ), + _system: Joi.any().meta({ internal: true }), + }) diff --git a/garden-service/src/plugins/kubernetes/container/build.ts b/garden-service/src/plugins/kubernetes/container/build.ts index 4b33e3ecb5d..4f7ed9730e4 100644 --- a/garden-service/src/plugins/kubernetes/container/build.ts +++ b/garden-service/src/plugins/kubernetes/container/build.ts @@ -9,25 +9,85 @@ import { ContainerModule } from "../../container/config" import { containerHelpers } from "../../container/helpers" import { buildContainerModule, getContainerBuildStatus } from "../../container/build" -import { GetBuildStatusParams } from "../../../types/plugin/module/getBuildStatus" -import { BuildModuleParams } from "../../../types/plugin/module/build" +import { GetBuildStatusParams, BuildStatus } from "../../../types/plugin/module/getBuildStatus" +import { BuildModuleParams, BuildResult } from "../../../types/plugin/module/build" +import { getPortForward, getPods } from "../util" +import { systemNamespace } from "../system" +import { RSYNC_PORT } from "../constants" +import execa = require("execa") +import { posix, resolve } from "path" +import { KubeApi } from "../api" +import { kubectl } from "../kubectl" +import { ConfigurationError } from "../../../exceptions" +import { LogEntry } from "../../../logger/log-entry" +import { KubernetesProvider } from "../config" -export async function getBuildStatus(params: GetBuildStatusParams) { - const status = await getContainerBuildStatus(params) +const builderDeployment = "garden-docker-daemon" +export async function k8sGetContainerBuildStatus( + params: GetBuildStatusParams, +): Promise { const { ctx } = params + const provider = ctx.provider - if (ctx.provider.config.deploymentRegistry) { - // TODO: Check if the image exists in the remote registry + if (provider.config.buildMode === "local") { + const status = await getContainerBuildStatus(params) + + if (ctx.provider.config.deploymentRegistry) { + // TODO: Check if the image exists in the remote registry + } + return status + + } else if (provider.config.buildMode === "cluster-docker") { + return getContainerBuildStatusCluster(params) + + } else { + throw invalidBuildMode(provider) } +} + +export async function k8sBuildContainer(params: BuildModuleParams): Promise { + const { ctx } = params + const provider = ctx.provider + + if (provider.config.buildMode === "local") { + return buildContainerLocal(params) - return status + } else if (provider.config.buildMode === "cluster-docker") { + return buildContainerCluster(params) + + } else { + throw invalidBuildMode(provider) + } } -export async function buildModule(params: BuildModuleParams) { - const buildResult = await buildContainerModule(params) +async function getContainerBuildStatusCluster(params: GetBuildStatusParams) { + const { ctx, module, log } = params + const provider = ctx.provider + + const hasDockerfile = await containerHelpers.hasDockerfile(module) + + if (!hasDockerfile) { + return { ready: true } + } + + const deploymentImage = await containerHelpers.getDeploymentImageId(module, provider.config.deploymentRegistry) + + const args = ["docker", "images", "-q", deploymentImage] + const res = await execInBuilder(provider, log, args, 30) + + const checkLog = res.stdout + res.stderr + log.silly(checkLog) + + // The `docker images -q ` command returns an ID if the image exists, otherwise it returns an empty string + const ready = checkLog.trim().length > 0 + + return { ready } +} +async function buildContainerLocal(params: BuildModuleParams) { const { ctx, module, log } = params + const buildResult = await buildContainerModule(params) if (!ctx.provider.config.deploymentRegistry) { return buildResult @@ -47,3 +107,105 @@ export async function buildModule(params: BuildModuleParams) { return buildResult } + +async function buildContainerCluster(params: BuildModuleParams) { + const { ctx, module, log } = params + const provider = ctx.provider + + const hasDockerfile = await containerHelpers.hasDockerfile(module) + + if (!hasDockerfile) { + log.setState("Nothing to build") + + return { + fetched: true, + fresh: false, + version: module.version.versionString, + } + } + + // Sync the build context to the remote sync service + // -> Get a tunnel to the service + log.setState("Syncing sources to cluster...") + const syncFwd = await getPortForward(ctx, log, systemNamespace, `Deployment/${builderDeployment}`, RSYNC_PORT) + + // -> Run rsync + const buildRoot = resolve(module.buildPath, "..") + // This trick is used to automatically create the correct target directory with rsync: + // https://stackoverflow.com/questions/1636889/rsync-how-can-i-configure-it-to-create-target-directory-on-server + const src = `${buildRoot}/./${module.name}/` + const destination = `rsync://localhost:${syncFwd.localPort}/volume/` + + log.debug(`Syncing from ${src} to ${destination}`) + // TODO: use list of files from module version + await execa("rsync", ["-vrpztgo", "--relative", src, destination]) + + // Execute the build + const localId = await containerHelpers.getLocalImageId(module) + const deploymentImageId = await containerHelpers.getDeploymentImageId(module, provider.config.deploymentRegistry) + + log.setState(`Building image ${localId}...`) + + // Prepare the build command + const dockerfile = module.spec.dockerfile || "Dockerfile" + const contextPath = `/garden-build/${module.name}` + const dockerfilePath = posix.join(contextPath, dockerfile) + + const buildArgs = [ + "docker", "build", + "-t", deploymentImageId, + "-f", dockerfilePath, + `/garden-build/${module.name}`, + ] + + const buildRes = await execInBuilder(provider, log, buildArgs, 600) + + const buildLog = buildRes.stdout + buildRes.stderr + log.silly(buildLog) + + // Push the image to the registry + log.setState({ msg: `Pushing image ${localId} to registry...` }) + + const dockerCmd = ["docker", "push", deploymentImageId] + const pushArgs = ["/bin/sh", "-c", dockerCmd.join(" ")] + + await execInBuilder(provider, log, pushArgs, 300) + + return { + buildLog, + fetched: false, + fresh: true, + version: module.version.versionString, + } +} + +// TODO: we should make a simple service around this instead of execing into containers +async function execInBuilder(provider: KubernetesProvider, log: LogEntry, args: string[], timeout: number) { + const api = await KubeApi.factory(log, provider.config.context) + const builderDockerPodName = await getBuilderPodName(api) + + const execCmd = ["exec", "-i", builderDockerPodName, "-c", "docker-daemon", "--", ...args] + + log.verbose(`Running: kubectl ${execCmd.join(" ")}`) + + return kubectl.exec({ + args: execCmd, + context: api.context, + log, + namespace: systemNamespace, + timeout, + }) +} + +async function getBuilderPodName(api: KubeApi) { + const builderStatusRes = await api.apps.readNamespacedDeployment(builderDeployment, systemNamespace) + const builderPods = await getPods(api, systemNamespace, builderStatusRes.body.spec.selector.matchLabels) + return builderPods[0].metadata.name +} + +function invalidBuildMode(provider: KubernetesProvider) { + return new ConfigurationError( + `kubernetes: Invalid build mode '${provider.config.buildMode}'`, + { config: provider.config }, + ) +} diff --git a/garden-service/src/plugins/kubernetes/container/deployment.ts b/garden-service/src/plugins/kubernetes/container/deployment.ts index dd44455b825..40c14aadf77 100644 --- a/garden-service/src/plugins/kubernetes/container/deployment.ts +++ b/garden-service/src/plugins/kubernetes/container/deployment.ts @@ -17,7 +17,7 @@ import { getAppNamespace } from "../namespace" import { PluginContext } from "../../../plugin-context" import { GARDEN_ANNOTATION_KEYS_VERSION } from "../../../constants" import { KubeApi } from "../api" -import { KubernetesProvider, KubernetesPluginContext } from "../kubernetes" +import { KubernetesProvider, KubernetesPluginContext } from "../config" import { configureHotReload } from "../hot-reload" import { KubernetesResource, KubeEnvVar } from "../types" import { ConfigurationError } from "../../../exceptions" diff --git a/garden-service/src/plugins/kubernetes/container/handlers.ts b/garden-service/src/plugins/kubernetes/container/handlers.ts index ef9e07c6112..4cebdf26e24 100644 --- a/garden-service/src/plugins/kubernetes/container/handlers.ts +++ b/garden-service/src/plugins/kubernetes/container/handlers.ts @@ -13,14 +13,14 @@ import { execInService, runContainerModule, runContainerService, runContainerTas import { testContainerModule } from "./test" import { ConfigurationError } from "../../../exceptions" import { configureContainerModule } from "../../container/container" -import { KubernetesProvider } from "../kubernetes" +import { KubernetesProvider } from "../config" import { ConfigureModuleParams } from "../../../types/plugin/module/configure" import { getContainerServiceStatus } from "./status" import { getTestResult } from "../test" import { ContainerModule } from "../../container/config" import { configureMavenContainerModule, MavenContainerModule } from "../../maven-container/maven-container" import { getTaskResult } from "../task-results" -import { buildModule, getBuildStatus } from "./build" +import { k8sBuildContainer, k8sGetContainerBuildStatus } from "./build" async function configure(params: ConfigureModuleParams) { params.moduleConfig = await configureContainerModule(params) @@ -35,11 +35,11 @@ export async function configureMaven(params: ConfigureModuleParams) { diff --git a/garden-service/src/plugins/kubernetes/container/run.ts b/garden-service/src/plugins/kubernetes/container/run.ts index f9a04ce95a4..f570fee9dbb 100644 --- a/garden-service/src/plugins/kubernetes/container/run.ts +++ b/garden-service/src/plugins/kubernetes/container/run.ts @@ -15,7 +15,7 @@ import { kubectl } from "../kubectl" import { getContainerServiceStatus } from "./status" import { runPod } from "../run" import { containerHelpers } from "../../container/helpers" -import { KubernetesPluginContext, KubernetesProvider } from "../kubernetes" +import { KubernetesPluginContext, KubernetesProvider } from "../config" import { storeTaskResult } from "../task-results" import { ExecInServiceParams } from "../../../types/plugin/service/execInService" import { RunModuleParams } from "../../../types/plugin/module/runModule" diff --git a/garden-service/src/plugins/kubernetes/container/status.ts b/garden-service/src/plugins/kubernetes/container/status.ts index 7dd3b1a3239..3f82472c2c7 100644 --- a/garden-service/src/plugins/kubernetes/container/status.ts +++ b/garden-service/src/plugins/kubernetes/container/status.ts @@ -19,7 +19,7 @@ import { KubeApi } from "../api" import { compareDeployedObjects } from "../status" import { getIngresses } from "./ingress" import { getAppNamespace } from "../namespace" -import { KubernetesPluginContext } from "../kubernetes" +import { KubernetesPluginContext } from "../config" export async function getContainerServiceStatus( { ctx, module, service, runtimeContext, log, hotReload }: GetServiceStatusParams, diff --git a/garden-service/src/plugins/kubernetes/helm/build.ts b/garden-service/src/plugins/kubernetes/helm/build.ts index fcd6c718c6c..e1f43e5968f 100644 --- a/garden-service/src/plugins/kubernetes/helm/build.ts +++ b/garden-service/src/plugins/kubernetes/helm/build.ts @@ -14,7 +14,7 @@ import { dumpYaml } from "../../../util/util" import { LogEntry } from "../../../logger/log-entry" import { getNamespace } from "../namespace" import { apply as jsonMerge } from "json-merge-patch" -import { KubernetesPluginContext } from "../kubernetes" +import { KubernetesPluginContext } from "../config" import { BuildModuleParams, BuildResult } from "../../../types/plugin/module/build" export async function buildHelmModule({ ctx, module, log }: BuildModuleParams): Promise { diff --git a/garden-service/src/plugins/kubernetes/helm/common.ts b/garden-service/src/plugins/kubernetes/helm/common.ts index d50cf207b95..56292cc1b15 100644 --- a/garden-service/src/plugins/kubernetes/helm/common.ts +++ b/garden-service/src/plugins/kubernetes/helm/common.ts @@ -25,7 +25,7 @@ import { Module } from "../../../types/module" import { findByName } from "../../../util/util" import { deline } from "../../../util/string" import { getAnnotation } from "../util" -import { KubernetesPluginContext } from "../kubernetes" +import { KubernetesPluginContext } from "../config" /** * Returns true if the specified Helm module contains a template (as opposed to just referencing a remote template). diff --git a/garden-service/src/plugins/kubernetes/helm/deployment.ts b/garden-service/src/plugins/kubernetes/helm/deployment.ts index bf8d8c55b66..1d27ffd575f 100644 --- a/garden-service/src/plugins/kubernetes/helm/deployment.ts +++ b/garden-service/src/plugins/kubernetes/helm/deployment.ts @@ -22,7 +22,7 @@ import { import { getReleaseStatus, getServiceStatus } from "./status" import { configureHotReload, HotReloadableResource } from "../hot-reload" import { apply } from "../kubectl" -import { KubernetesPluginContext } from "../kubernetes" +import { KubernetesPluginContext } from "../config" import { ContainerHotReloadSpec } from "../../container/config" import { getHotReloadSpec } from "./hot-reload" import { DeployServiceParams } from "../../../types/plugin/service/deployService" diff --git a/garden-service/src/plugins/kubernetes/helm/hot-reload.ts b/garden-service/src/plugins/kubernetes/helm/hot-reload.ts index bd00c05bf55..2d3c895a1f9 100644 --- a/garden-service/src/plugins/kubernetes/helm/hot-reload.ts +++ b/garden-service/src/plugins/kubernetes/helm/hot-reload.ts @@ -12,7 +12,7 @@ import { deline } from "../../../util/string" import { ContainerModule } from "../../container/config" import { getChartResources, findServiceResource, getServiceResourceSpec } from "./common" import { syncToService, HotReloadableKind } from "../hot-reload" -import { KubernetesPluginContext } from "../kubernetes" +import { KubernetesPluginContext } from "../config" import { HotReloadServiceParams, HotReloadServiceResult } from "../../../types/plugin/service/hotReloadService" /** diff --git a/garden-service/src/plugins/kubernetes/helm/logs.ts b/garden-service/src/plugins/kubernetes/helm/logs.ts index 721097336b4..b7e95806417 100644 --- a/garden-service/src/plugins/kubernetes/helm/logs.ts +++ b/garden-service/src/plugins/kubernetes/helm/logs.ts @@ -10,7 +10,7 @@ import { GetServiceLogsParams } from "../../../types/plugin/service/getServiceLo import { getAppNamespace } from "../namespace" import { getAllLogs } from "../logs" import { HelmModule } from "./config" -import { KubernetesPluginContext } from "../kubernetes" +import { KubernetesPluginContext } from "../config" import { getChartResources } from "./common" export async function getServiceLogs(params: GetServiceLogsParams) { diff --git a/garden-service/src/plugins/kubernetes/helm/run.ts b/garden-service/src/plugins/kubernetes/helm/run.ts index 9cee5fc34bc..23ed219dfe9 100644 --- a/garden-service/src/plugins/kubernetes/helm/run.ts +++ b/garden-service/src/plugins/kubernetes/helm/run.ts @@ -13,7 +13,7 @@ import { findServiceResource, getChartResources, getResourceContainer, getServic import { PluginContext } from "../../../plugin-context" import { LogEntry } from "../../../logger/log-entry" import { ConfigurationError } from "../../../exceptions" -import { KubernetesPluginContext } from "../kubernetes" +import { KubernetesPluginContext } from "../config" import { storeTaskResult } from "../task-results" import { RunModuleParams } from "../../../types/plugin/module/runModule" import { RunResult } from "../../../types/plugin/base" diff --git a/garden-service/src/plugins/kubernetes/helm/status.ts b/garden-service/src/plugins/kubernetes/helm/status.ts index 78728bb6319..15cdebc3739 100644 --- a/garden-service/src/plugins/kubernetes/helm/status.ts +++ b/garden-service/src/plugins/kubernetes/helm/status.ts @@ -19,7 +19,7 @@ import { getChartResources, findServiceResource } from "./common" import { buildHelmModule } from "./build" import { configureHotReload } from "../hot-reload" import { getHotReloadSpec } from "./hot-reload" -import { KubernetesPluginContext } from "../kubernetes" +import { KubernetesPluginContext } from "../config" const helmStatusCodeMap: { [code: number]: ServiceState } = { // see https://github.com/kubernetes/helm/blob/master/_proto/hapi/release/status.proto diff --git a/garden-service/src/plugins/kubernetes/helm/test.ts b/garden-service/src/plugins/kubernetes/helm/test.ts index 778977043b2..7530697a36a 100644 --- a/garden-service/src/plugins/kubernetes/helm/test.ts +++ b/garden-service/src/plugins/kubernetes/helm/test.ts @@ -12,7 +12,7 @@ import { HelmModule } from "./config" import { getAppNamespace } from "../namespace" import { runPod } from "../run" import { findServiceResource, getChartResources, getResourceContainer, getServiceResourceSpec } from "./common" -import { KubernetesPluginContext } from "../kubernetes" +import { KubernetesPluginContext } from "../config" import { TestModuleParams } from "../../../types/plugin/module/testModule" import { TestResult } from "../../../types/plugin/module/getTestResult" diff --git a/garden-service/src/plugins/kubernetes/helm/tiller.ts b/garden-service/src/plugins/kubernetes/helm/tiller.ts index 609d1047075..ac1dd028265 100644 --- a/garden-service/src/plugins/kubernetes/helm/tiller.ts +++ b/garden-service/src/plugins/kubernetes/helm/tiller.ts @@ -16,7 +16,7 @@ import { getAppNamespace } from "../namespace" import { checkResourceStatuses, waitForResources } from "../status" import { combineStates } from "../../../types/service" import { apply } from "../kubectl" -import { KubernetesProvider } from "../kubernetes" +import { KubernetesProvider } from "../config" import chalk from "chalk" const serviceAccountName = "garden-tiller" diff --git a/garden-service/src/plugins/kubernetes/hot-reload.ts b/garden-service/src/plugins/kubernetes/hot-reload.ts index 988e0c43653..0883fb7601f 100644 --- a/garden-service/src/plugins/kubernetes/hot-reload.ts +++ b/garden-service/src/plugins/kubernetes/hot-reload.ts @@ -22,7 +22,7 @@ import { waitForContainerService } from "./container/status" import { getPortForward } from "./util" import { RSYNC_PORT } from "./constants" import { getAppNamespace } from "./namespace" -import { KubernetesPluginContext } from "./kubernetes" +import { KubernetesPluginContext } from "./config" import { HotReloadServiceParams, HotReloadServiceResult } from "../../types/plugin/service/hotReloadService" export const RSYNC_PORT_NAME = "garden-rsync" diff --git a/garden-service/src/plugins/kubernetes/init.ts b/garden-service/src/plugins/kubernetes/init.ts index 320633f6804..2d25371d66b 100644 --- a/garden-service/src/plugins/kubernetes/init.ts +++ b/garden-service/src/plugins/kubernetes/init.ts @@ -8,24 +8,20 @@ import { KubeApi } from "./api" import { getAppNamespace, prepareNamespaces, deleteNamespaces } from "./namespace" -import { KubernetesPluginContext } from "./kubernetes" +import { KubernetesPluginContext, KubernetesConfig } from "./config" import { checkTillerStatus, installTiller } from "./helm/tiller" import { prepareSystemServices, getSystemServiceStatuses, getSystemGarden, systemNamespaceUpToDate, + systemNamespace, } from "./system" -import { PrimitiveMap } from "../../config/common" import { DashboardPage } from "../../config/dashboard" import { GetEnvironmentStatusParams, EnvironmentStatus } from "../../types/plugin/provider/getEnvironmentStatus" import { PrepareEnvironmentParams } from "../../types/plugin/provider/prepareEnvironment" import { CleanupEnvironmentParams } from "../../types/plugin/provider/cleanupEnvironment" -interface GetK8sEnvironmentStatusParams extends GetEnvironmentStatusParams { - variables?: PrimitiveMap -} - /** * Performs the following actions to check environment status: * 1. Checks Tiller status in the project namespace @@ -34,10 +30,10 @@ interface GetK8sEnvironmentStatusParams extends GetEnvironmentStatusParams { * * Returns ready === true if all the above are ready. */ -export async function getEnvironmentStatus( - { ctx, log, variables }: GetK8sEnvironmentStatusParams, -): Promise { +export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusParams): Promise { const k8sCtx = ctx + const variables = getVariables(k8sCtx.provider.config) + const sysGarden = await getSystemGarden(k8sCtx.provider, variables || {}) const sysCtx = await sysGarden.getPluginContext(k8sCtx.provider.name) @@ -91,18 +87,15 @@ export async function getEnvironmentStatus( } } -interface PrepareK8sEnvironmentParams extends PrepareEnvironmentParams { - variables?: PrimitiveMap -} - /** * Performs the following actions to prepare the environment * 1. Installs Tiller in project namespace * 2. Installs Tiller in system namespace (if provider has system services) * 3. Deploys system services (if provider has system services) */ -export async function prepareEnvironment({ ctx, log, force, status, variables }: PrepareK8sEnvironmentParams) { +export async function prepareEnvironment({ ctx, log, force, status }: PrepareEnvironmentParams) { const k8sCtx = ctx + const variables = getVariables(k8sCtx.provider.config) const systemReady = status.detail && !!status.detail.systemReady && !force // Install Tiller to project namespace @@ -146,3 +139,26 @@ export async function cleanupEnvironment({ ctx, log }: CleanupEnvironmentParams) return {} } + +function getVariables(config: KubernetesConfig) { + return { + "namespace": systemNamespace, + "registry-hostname": getRegistryHostname(), + "builder-limits-cpu": config.resources.builder.limits.cpu, + "builder-limits-memory": config.resources.builder.limits.memory, + "builder-requests-cpu": config.resources.builder.requests.cpu, + "builder-requests-memory": config.resources.builder.requests.memory, + "builder-storage-size": config.storage.builder.size, + "builder-storage-class": config.storage.builder.storageClass, + "registry-limits-cpu": config.resources.registry.limits.cpu, + "registry-limits-memory": config.resources.registry.limits.memory, + "registry-requests-cpu": config.resources.registry.requests.cpu, + "registry-requests-memory": config.resources.registry.requests.memory, + "registry-storage-size": config.storage.registry.size, + "registry-storage-class": config.storage.registry.storageClass, + } +} + +function getRegistryHostname() { + return `garden-docker-registry.${systemNamespace}.svc.cluster.local` +} diff --git a/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts b/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts index 34d6e860679..92210e12709 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts @@ -14,7 +14,7 @@ import { safeLoadAll } from "js-yaml" import { KubernetesModule, configureKubernetesModule, KubernetesService, describeType } from "./config" import { getNamespace, getAppNamespace } from "../namespace" -import { KubernetesPluginContext } from "../kubernetes" +import { KubernetesPluginContext } from "../config" import { KubernetesResource } from "../types" import { ServiceStatus } from "../../../types/service" import { GARDEN_ANNOTATION_KEYS_SERVICE } from "../../../constants" diff --git a/garden-service/src/plugins/kubernetes/kubernetes.ts b/garden-service/src/plugins/kubernetes/kubernetes.ts index c4e0fcd7c57..d9643e15e62 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes.ts @@ -6,155 +6,54 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import * as Joi from "joi" import * as Bluebird from "bluebird" -import dedent = require("dedent") -import { joiArray, joiIdentifier, joiProviderName } from "../../config/common" import { GardenPlugin } from "../../types/plugin/plugin" -import { Provider, providerConfigBaseSchema, ProviderConfig } from "../../config/project" import { helmHandlers } from "./helm/handlers" import { getAppNamespace, getMetadataNamespace } from "./namespace" import { getSecret, setSecret, deleteSecret } from "./secrets" -import { containerRegistryConfigSchema, ContainerRegistryConfig } from "../container/config" import { getEnvironmentStatus, prepareEnvironment, cleanupEnvironment } from "./init" import { containerHandlers, mavenContainerHandlers } from "./container/handlers" -import { PluginContext } from "../../plugin-context" import { kubernetesHandlers } from "./kubernetes-module/handlers" import { ConfigureProviderParams } from "../../types/plugin/provider/configureProvider" import { DebugInfo, GetDebugInfoParams } from "../../types/plugin/provider/getDebugInfo" import { systemNamespace, systemMetadataNamespace } from "./system" import { kubectl } from "./kubectl" +import { KubernetesConfig, KubernetesPluginContext } from "./config" +import { configSchema } from "../../config/base" +import { ConfigurationError } from "../../exceptions" export const name = "kubernetes" -export interface SecretRef { - name: string - namespace: string -} - -export interface IngressTlsCertificate { - name: string - hostnames?: string[] - secretRef: SecretRef -} - -export interface KubernetesBaseConfig extends ProviderConfig { - context: string - defaultHostname?: string - defaultUsername?: string - forceSsl: boolean - imagePullSecrets: SecretRef[] - ingressHttpPort: number - ingressHttpsPort: number - ingressClass?: string - namespace?: string - tlsCertificates: IngressTlsCertificate[] - _systemServices: string[] -} - -export interface KubernetesConfig extends KubernetesBaseConfig { - deploymentRegistry?: ContainerRegistryConfig -} - -export type KubernetesProvider = Provider -export type KubernetesPluginContext = PluginContext - -export const k8sContextSchema = Joi.string() - .required() - .description("The kubectl context to use to connect to the Kubernetes cluster.") - .example("my-dev-context") - -const secretRef = Joi.object() - .keys({ - name: joiIdentifier() - .required() - .description("The name of the Kubernetes secret.") - .example("my-secret"), - namespace: joiIdentifier() - .default("default") - .description( - "The namespace where the secret is stored. " + - "If necessary, the secret may be copied to the appropriate namespace before use.", - ), - }) - .description("Reference to a Kubernetes secret.") - -const imagePullSecretsSchema = joiArray(secretRef) - .description(dedent` - References to \`docker-registry\` secrets to use for authenticating with remote registries when pulling - images. This is necessary if you reference private images in your module configuration, and is required - when configuring a remote Kubernetes environment. - `) +export async function configureProvider({ projectName, config }: ConfigureProviderParams) { + if (!config.namespace) { + config.namespace = projectName + } -const tlsCertificateSchema = Joi.object() - .keys({ - name: joiIdentifier() - .required() - .description("A unique identifier for this certificate.") - .example("www") - .example("wildcard"), - hostnames: Joi.array().items(Joi.string().hostname()) - .description( - "A list of hostnames that this certificate should be used for. " + - "If you don't specify these, they will be automatically read from the certificate.", + if (config.buildMode === "cluster-docker") { + if (config.deploymentRegistry) { + throw new ConfigurationError( + `kubernetes: deploymentRegistry should not be set in config if using cluster-docker build mode`, + { config }, ) - .example([["www.mydomain.com"], {}]), - secretRef: secretRef - .description("A reference to the Kubernetes secret that contains the TLS certificate and key for the domain.") - .example({ name: "my-tls-secret", namespace: "default" }), - }) + } -export const kubernetesConfigBase = providerConfigBaseSchema - .keys({ - defaultHostname: Joi.string() - .description("A default hostname to use when no hostname is explicitly configured for a service.") - .example("api.mydomain.com"), - defaultUsername: joiIdentifier() - .description("Set a default username (used for namespacing within a cluster)."), - forceSsl: Joi.boolean() - .default(false) - .description( - "Require SSL on all services. If set to true, an error is raised when no certificate " + - "is available for a configured hostname.", - ), - imagePullSecrets: imagePullSecretsSchema, - tlsCertificates: joiArray(tlsCertificateSchema) - .unique("name") - .description("One or more certificates to use for ingress."), - _systemServices: joiArray(joiIdentifier()) - .meta({ internal: true }), - }) + // This is a special configuration, used in combination with the registry-proxy service, + // to make sure every node in the cluster can resolve the image from the registry we deploy in-cluster. + config.deploymentRegistry = { + hostname: `127.0.0.1:5000`, + // The base configure handler ensures that the namespace is set + namespace: config.namespace!, + } -export const configSchema = kubernetesConfigBase - .keys({ - name: joiProviderName("kubernetes"), - context: k8sContextSchema - .required(), - deploymentRegistry: containerRegistryConfigSchema, - ingressClass: Joi.string() - .description(dedent` - The ingress class to use on configured Ingresses (via the \`kubernetes.io/ingress.class\` annotation) - when deploying \`container\` services. Use this if you have multiple ingress controllers in your cluster. - `), - ingressHttpPort: Joi.number() - .default(80) - .description("The external HTTP port of the cluster's ingress controller."), - ingressHttpsPort: Joi.number() - .default(443) - .description("The external HTTPS port of the cluster's ingress controller."), - namespace: Joi.string() - .default(undefined, "") - .description( - "Specify which namespace to deploy services to (defaults to ). " + - "Note that the framework generates other namespaces as well with this name as a prefix.", - ), - _system: Joi.any().meta({ internal: true }), - }) + // Deploy build services on init + config._systemServices = ["docker-daemon", "docker-registry", "registry-proxy"] -export async function configureProvider({ projectName, config }: ConfigureProviderParams) { - if (!config.namespace) { - config.namespace = projectName + } else if (!config.deploymentRegistry) { + throw new ConfigurationError( + `kubernetes: must specify deploymentRegistry in config if using local build mode`, + { config }, + ) } return { name: config.name, config } diff --git a/garden-service/src/plugins/kubernetes/local/config.ts b/garden-service/src/plugins/kubernetes/local/config.ts index 147bfdda493..435b2dce387 100644 --- a/garden-service/src/plugins/kubernetes/local/config.ts +++ b/garden-service/src/plugins/kubernetes/local/config.ts @@ -8,7 +8,7 @@ import * as execa from "execa" import * as Joi from "joi" -import { KubernetesBaseConfig, kubernetesConfigBase, k8sContextSchema } from "../kubernetes" +import { KubernetesBaseConfig, kubernetesConfigBase, k8sContextSchema } from "../config" import { ConfigureProviderParams } from "../../../types/plugin/provider/configureProvider" import { joiProviderName } from "../../../config/common" import { getKubeConfig } from "../api" @@ -130,6 +130,7 @@ export async function configureProvider({ config, log, projectName }: ConfigureP config = { name: config.name, + buildMode: config.buildMode, context, defaultHostname, deploymentRegistry, @@ -139,6 +140,8 @@ export async function configureProvider({ config, log, projectName }: ConfigureP ingressHttpsPort: 443, ingressClass, namespace, + resources: config.resources, + storage: config.storage, setupIngressController: config.setupIngressController, tlsCertificates: config.tlsCertificates, _system: config._system, diff --git a/garden-service/src/plugins/kubernetes/namespace.ts b/garden-service/src/plugins/kubernetes/namespace.ts index e2dad682c5c..55fc6b2935b 100644 --- a/garden-service/src/plugins/kubernetes/namespace.ts +++ b/garden-service/src/plugins/kubernetes/namespace.ts @@ -11,7 +11,7 @@ import { intersection } from "lodash" import { PluginContext } from "../../plugin-context" import { KubeApi } from "./api" -import { KubernetesProvider, KubernetesPluginContext } from "./kubernetes" +import { KubernetesProvider, KubernetesPluginContext } from "./config" import { name as providerName } from "./kubernetes" import { AuthenticationError, DeploymentError, TimeoutError } from "../../exceptions" import { getPackageVersion, sleep } from "../../util/util" diff --git a/garden-service/src/plugins/kubernetes/secrets.ts b/garden-service/src/plugins/kubernetes/secrets.ts index 6058fb61e1b..76c6ba4cc79 100644 --- a/garden-service/src/plugins/kubernetes/secrets.ts +++ b/garden-service/src/plugins/kubernetes/secrets.ts @@ -9,7 +9,7 @@ import { V1Secret } from "@kubernetes/client-node" import { KubeApi } from "./api" -import { SecretRef, KubernetesPluginContext } from "./kubernetes" +import { SecretRef, KubernetesPluginContext } from "./config" import { ConfigurationError } from "../../exceptions" import { getMetadataNamespace } from "./namespace" import { GetSecretParams } from "../../types/plugin/provider/getSecret" diff --git a/garden-service/src/plugins/kubernetes/status.ts b/garden-service/src/plugins/kubernetes/status.ts index 21561a31b69..eb21090d748 100644 --- a/garden-service/src/plugins/kubernetes/status.ts +++ b/garden-service/src/plugins/kubernetes/status.ts @@ -26,7 +26,7 @@ import { V1DeploymentStatus, } from "@kubernetes/client-node" import { some, zip, isArray, isPlainObject, pickBy, mapValues } from "lodash" -import { KubernetesProvider, KubernetesPluginContext } from "./kubernetes" +import { KubernetesProvider, KubernetesPluginContext } from "./config" import { isSubset } from "../../util/is-subset" import { LogEntry } from "../../logger/log-entry" import { V1ReplicationController, V1ReplicaSet } from "@kubernetes/client-node" diff --git a/garden-service/src/plugins/kubernetes/system.ts b/garden-service/src/plugins/kubernetes/system.ts index a62afc39e20..e0e2806adf7 100644 --- a/garden-service/src/plugins/kubernetes/system.ts +++ b/garden-service/src/plugins/kubernetes/system.ts @@ -13,7 +13,7 @@ import * as semver from "semver" import { STATIC_DIR } from "../../constants" import { Garden } from "../../garden" -import { KubernetesProvider, KubernetesPluginContext } from "./kubernetes" +import { KubernetesProvider, KubernetesPluginContext } from "./config" import { LogEntry } from "../../logger/log-entry" import { KubeApi } from "./api" import { createNamespace } from "./namespace" diff --git a/garden-service/src/plugins/kubernetes/task-results.ts b/garden-service/src/plugins/kubernetes/task-results.ts index f2f1a2b2f78..0f1fc2ac30f 100644 --- a/garden-service/src/plugins/kubernetes/task-results.ts +++ b/garden-service/src/plugins/kubernetes/task-results.ts @@ -10,7 +10,7 @@ import { GetTaskResultParams } from "../../types/plugin/task/getTaskResult" import { ContainerModule } from "../container/config" import { HelmModule } from "./helm/config" import { ModuleVersion } from "../../vcs/vcs" -import { KubernetesPluginContext, KubernetesProvider } from "./kubernetes" +import { KubernetesPluginContext, KubernetesProvider } from "./config" import { KubeApi } from "./api" import { getMetadataNamespace } from "./namespace" import { RunTaskResult } from "../../types/plugin/task/runTask" diff --git a/garden-service/src/plugins/kubernetes/test.ts b/garden-service/src/plugins/kubernetes/test.ts index d86ad2abfc1..a044877bce0 100644 --- a/garden-service/src/plugins/kubernetes/test.ts +++ b/garden-service/src/plugins/kubernetes/test.ts @@ -13,7 +13,7 @@ import { Module } from "../../types/module" import { ModuleVersion } from "../../vcs/vcs" import { HelmModule } from "./helm/config" import { PluginContext } from "../../plugin-context" -import { KubernetesPluginContext } from "./kubernetes" +import { KubernetesPluginContext } from "./config" import { systemMetadataNamespace } from "./system" import { LogEntry } from "../../logger/log-entry" import { GetTestResultParams, TestResult } from "../../types/plugin/module/getTestResult" diff --git a/garden-service/src/plugins/kubernetes/util.ts b/garden-service/src/plugins/kubernetes/util.ts index 028c2805adf..c0179991e8a 100644 --- a/garden-service/src/plugins/kubernetes/util.ts +++ b/garden-service/src/plugins/kubernetes/util.ts @@ -17,7 +17,7 @@ import { splitLast } from "../../util/util" import { KubeApi } from "./api" import { PluginContext } from "../../plugin-context" import { LogEntry } from "../../logger/log-entry" -import { KubernetesPluginContext } from "./kubernetes" +import { KubernetesPluginContext } from "./config" import { kubectl } from "./kubectl" import { registerCleanupFunction } from "../../util/util" diff --git a/garden-service/src/plugins/openfaas/openfaas.ts b/garden-service/src/plugins/openfaas/openfaas.ts index 8b685cb0c11..85e6ef4b57b 100644 --- a/garden-service/src/plugins/openfaas/openfaas.ts +++ b/garden-service/src/plugins/openfaas/openfaas.ts @@ -23,7 +23,7 @@ import { testExecModule, getExecModuleBuildStatus, } from "../exec" -import { KubernetesProvider } from "../kubernetes/kubernetes" +import { KubernetesProvider } from "../kubernetes/config" import { getNamespace, getAppNamespace } from "../kubernetes/namespace" import { every, values } from "lodash" import { dumpYaml, findByName } from "../../util/util" diff --git a/garden-service/static/kubernetes/system/docker-daemon/Chart.yaml b/garden-service/static/kubernetes/system/docker-daemon/Chart.yaml new file mode 100644 index 00000000000..f29ca00d227 --- /dev/null +++ b/garden-service/static/kubernetes/system/docker-daemon/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: docker-daemon +version: 0.1.0 diff --git a/garden-service/static/kubernetes/system/docker-daemon/garden.yml b/garden-service/static/kubernetes/system/docker-daemon/garden.yml new file mode 100644 index 00000000000..0097aeede20 --- /dev/null +++ b/garden-service/static/kubernetes/system/docker-daemon/garden.yml @@ -0,0 +1,23 @@ +kind: Module +type: helm +name: docker-daemon +description: Docker daemon used for in-cluster building +releaseName: garden-docker-daemon +values: + dockerDaemon: + resources: + limits: + cpu: ${var.builder-limits-cpu} + memory: ${var.builder-limits-memory} + requests: + cpu: ${var.builder-requests-cpu} + memory: ${var.builder-requests-memory} + registry: + hostname: ${var.registry-hostname || "foo"} + # tlsSecretName: ${variables.registry-tls-secret-name} + storage: + size: ${var.builder-storage-size} + storageClass: ${var.builder-storage-class} + sync: + storage: + storageClass: ${var.builder-storage-class} diff --git a/garden-service/static/kubernetes/system/docker-daemon/templates/NOTES.txt b/garden-service/static/kubernetes/system/docker-daemon/templates/NOTES.txt new file mode 100644 index 00000000000..fce3669dba9 --- /dev/null +++ b/garden-service/static/kubernetes/system/docker-daemon/templates/NOTES.txt @@ -0,0 +1,15 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "docker-daemon.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "docker-daemon.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "docker-daemon.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "docker-daemon.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward $POD_NAME 8080:80 +{{- end }} diff --git a/garden-service/static/kubernetes/system/docker-daemon/templates/_helpers.tpl b/garden-service/static/kubernetes/system/docker-daemon/templates/_helpers.tpl new file mode 100644 index 00000000000..ea5b5b94dfe --- /dev/null +++ b/garden-service/static/kubernetes/system/docker-daemon/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "docker-daemon.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "docker-daemon.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "docker-daemon.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/garden-service/static/kubernetes/system/docker-daemon/templates/deployment.yaml b/garden-service/static/kubernetes/system/docker-daemon/templates/deployment.yaml new file mode 100644 index 00000000000..366d9ab5822 --- /dev/null +++ b/garden-service/static/kubernetes/system/docker-daemon/templates/deployment.yaml @@ -0,0 +1,115 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "docker-daemon.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "docker-daemon.name" . }} + helm.sh/chart: {{ include "docker-daemon.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ include "docker-daemon.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + strategy: + # We only want one instance at a time, because we're using the same volume for the data + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "docker-daemon.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + volumes: + - name: garden-docker-data + persistentVolumeClaim: + claimName: garden-docker-data + - name: garden-build-sync-data + persistentVolumeClaim: + claimName: garden-build-sync-data + # - name: garden-registry-tls + # secret: + # secretName: {{ .Values.dockerDaemon.registry.tlsSecretName }} + # items: + # - key: tls.crt + # path: localhost:5000/ca.crt + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: docker + containerPort: 2375 + protocol: TCP + securityContext: + privileged: true + livenessProbe: + tcpSocket: + port: 2375 + readinessProbe: + tcpSocket: + port: 2375 + volumeMounts: + - name: garden-docker-data + mountPath: /var/lib/docker + - name: garden-build-sync-data + mountPath: /garden-build + # Need to mount the registry cert so that the daemon trusts it + # - name: garden-registry-tls + # mountPath: /etc/docker/certs.d + resources: + {{- toYaml .Values.dockerDaemon.resources | nindent 12 }} + - name: {{ .Chart.Name }}-sync + image: "eugenmayer/rsync:latest" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: docker + containerPort: 873 + protocol: TCP + livenessProbe: + tcpSocket: + port: 873 + readinessProbe: + tcpSocket: + port: 873 + volumeMounts: + - mountPath: /data + name: garden-build-sync-data + env: + # The service is not exposed at all outside the cluster, so this should be all good. + - name: ALLOW + value: "0.0.0.0/0" + resources: + {{- toYaml .Values.sync.resources | nindent 12 }} + - name: {{ .Chart.Name }}-proxy + image: "basi/socat:v0.1.0" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + socat -d -d TCP-LISTEN:5000,fork TCP:{{ .Values.dockerDaemon.registry.hostname }}:5000 + ports: + - name: proxy + containerPort: 5000 + protocol: TCP + livenessProbe: + tcpSocket: + port: 5000 + readinessProbe: + tcpSocket: + port: 5000 + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/garden-service/static/kubernetes/system/docker-daemon/templates/service.yaml b/garden-service/static/kubernetes/system/docker-daemon/templates/service.yaml new file mode 100644 index 00000000000..d4127c51a14 --- /dev/null +++ b/garden-service/static/kubernetes/system/docker-daemon/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "docker-daemon.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "docker-daemon.name" . }} + helm.sh/chart: {{ include "docker-daemon.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: 2375 + protocol: TCP + name: docker + selector: + app.kubernetes.io/name: {{ include "docker-daemon.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/garden-service/static/kubernetes/system/docker-daemon/templates/volume.yaml b/garden-service/static/kubernetes/system/docker-daemon/templates/volume.yaml new file mode 100644 index 00000000000..988cd74b785 --- /dev/null +++ b/garden-service/static/kubernetes/system/docker-daemon/templates/volume.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: garden-docker-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.dockerDaemon.storage.request }} +{{- if .Values.dockerDaemon.storage.storageClass }} +{{- if (eq "-" .Values.dockerDaemon.storage.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.dockerDaemon.storage.storageClass }}" +{{- end }} +{{- end }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: garden-build-sync-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.sync.storage.request }} +{{- if .Values.sync.storage.storageClass }} +{{- if (eq "-" .Values.sync.storage.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.sync.storage.storageClass }}" +{{- end }} +{{- end }} \ No newline at end of file diff --git a/garden-service/static/kubernetes/system/docker-daemon/values.yaml b/garden-service/static/kubernetes/system/docker-daemon/values.yaml new file mode 100644 index 00000000000..eedffc1af42 --- /dev/null +++ b/garden-service/static/kubernetes/system/docker-daemon/values.yaml @@ -0,0 +1,45 @@ +# Default values for docker-daemon. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: docker + tag: 18.09.3-dind + pullPolicy: IfNotPresent + +nameOverride: "garden-docker-daemon" +fullnameOverride: "garden-docker-daemon" + +service: + type: ClusterIP + port: 2375 + +dockerDaemon: + resources: + limits: + cpu: "2" + memory: 4Gi + requests: + cpu: 200m + memory: 256Mi + storage: + request: 20Gi + registry: + hostname: garden-docker-registry + +sync: + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + storage: + request: 2Gi + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/garden-service/static/kubernetes/system/docker-registry/garden.yml b/garden-service/static/kubernetes/system/docker-registry/garden.yml new file mode 100644 index 00000000000..beb2d8c8b31 --- /dev/null +++ b/garden-service/static/kubernetes/system/docker-registry/garden.yml @@ -0,0 +1,23 @@ +kind: Module +description: Docker registry for in-cluster builds +name: docker-registry +type: helm +chart: stable/docker-registry +releaseName: garden-docker-registry +version: 1.7.0 +values: + resources: + limits: + cpu: ${var.registry-limits-cpu} + memory: ${var.registry-limits-memory} + requests: + cpu: ${var.registry-requests-cpu} + memory: ${var.registry-requests-memory} + # Note: this secret doesn't actually matter for security since the registry isn't exposed + secrets: + haSharedSecret: "TDVGTm45dVpCMXptOEJGMA==" + service: + port: 5000 + storage: + size: ${var.registry-storage-size} + storageClass: ${var.registry-storage-class} diff --git a/garden-service/static/kubernetes/system/registry-proxy/Chart.yaml b/garden-service/static/kubernetes/system/registry-proxy/Chart.yaml new file mode 100644 index 00000000000..f29ca00d227 --- /dev/null +++ b/garden-service/static/kubernetes/system/registry-proxy/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: docker-daemon +version: 0.1.0 diff --git a/garden-service/static/kubernetes/system/registry-proxy/garden.yml b/garden-service/static/kubernetes/system/registry-proxy/garden.yml new file mode 100644 index 00000000000..fc907f61c2a --- /dev/null +++ b/garden-service/static/kubernetes/system/registry-proxy/garden.yml @@ -0,0 +1,10 @@ +kind: Module +type: helm +name: registry-proxy +description: DaemonSet that proxies connections to the docker registry service on each node +releaseName: garden-registry-proxy +values: + dockerDaemon: + registry: + hostname: ${var.registry-hostname || "foo"} + # tlsSecretName: ${variables.registry-tls-secret-name} diff --git a/garden-service/static/kubernetes/system/registry-proxy/templates/_helpers.tpl b/garden-service/static/kubernetes/system/registry-proxy/templates/_helpers.tpl new file mode 100644 index 00000000000..ea5b5b94dfe --- /dev/null +++ b/garden-service/static/kubernetes/system/registry-proxy/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "docker-daemon.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "docker-daemon.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "docker-daemon.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/garden-service/static/kubernetes/system/registry-proxy/templates/daemonset.yaml b/garden-service/static/kubernetes/system/registry-proxy/templates/daemonset.yaml new file mode 100644 index 00000000000..119c870babf --- /dev/null +++ b/garden-service/static/kubernetes/system/registry-proxy/templates/daemonset.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "docker-daemon.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "docker-daemon.name" . }} + helm.sh/chart: {{ include "docker-daemon.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "docker-daemon.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + updateStrategy: + rollingUpdate: + maxUnavailable: 80% + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "docker-daemon.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + volumes: + # - name: docker-certs + # hostPath: + # path: /etc/docker/certs.d + # - name: garden-registry-tls + # secret: + # secretName: {{ .Values.dockerDaemon.registry.tlsSecretName }} + # items: + # - key: tls.crt + # path: ca.crt + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + # Copy the registry certs to the host + # cp -r /certs/localhost /etc/docker/certs.d/localhost:5000 + # Proxy node connections on 127.0.0.1:5000 to the docker registry + socat -d -d -d TCP-LISTEN:5000,fork,range=10.0.0.0/8 TCP:{{ .Values.dockerDaemon.registry.hostname }}:5000 + ports: + - name: docker + containerPort: 5000 + hostPort: 5000 + protocol: TCP + # securityContext: + # privileged: true + # livenessProbe: + # tcpSocket: + # port: 5000 + # readinessProbe: + # tcpSocket: + # port: 5000 + # volumeMounts: + # Need to add the registry cert to the node so that the daemon trusts it + # - name: docker-certs + # mountPath: "/etc/docker/certs.d" + # - name: garden-registry-tls + # mountPath: "/certs/localhost" + resources: + {{- toYaml .Values.dockerDaemon.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/garden-service/static/kubernetes/system/registry-proxy/values.yaml b/garden-service/static/kubernetes/system/registry-proxy/values.yaml new file mode 100644 index 00000000000..75ff85eb789 --- /dev/null +++ b/garden-service/static/kubernetes/system/registry-proxy/values.yaml @@ -0,0 +1,22 @@ +# Default values for docker-daemon. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: basi/socat + tag: v0.1.0 + pullPolicy: IfNotPresent + +nameOverride: garden-registry-proxy +fullnameOverride: garden-registry-proxy + +dockerDaemon: + registry: + hostname: garden-docker-registry + tlsSecretName: garden-docker-registry-tls + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts index 9d720bc6e11..9158d93e0ac 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts @@ -3,7 +3,12 @@ import { resolve, join } from "path" import * as td from "testdouble" import { KubeApi } from "../../../../../../src/plugins/kubernetes/api" -import { KubernetesProvider, KubernetesConfig } from "../../../../../../src/plugins/kubernetes/kubernetes" +import { + KubernetesProvider, + KubernetesConfig, + defaultResources, + defaultStorage, +} from "../../../../../../src/plugins/kubernetes/config" import { gardenPlugin } from "../../../../../../src/plugins/container/container" import { dataDir, makeTestGarden, expectError } from "../../../../../helpers" import { Garden } from "../../../../../../src/garden" @@ -27,6 +32,7 @@ const ports = [{ const basicConfig: KubernetesConfig = { name: "kubernetes", + buildMode: "local", context: "my-cluster", defaultHostname: "my.domain.com", deploymentRegistry: { @@ -39,6 +45,8 @@ const basicConfig: KubernetesConfig = { ingressClass: "nginx", ingressHttpPort: 80, ingressHttpsPort: 443, + resources: defaultResources, + storage: defaultStorage, tlsCertificates: [], _systemServices: [], }