fix(k8s): kaniko would hang when building remote images

.circleci/config.yml

@@ -315,7 +315,7 @@ jobs:
       - run:
           name: Deploy demo-project with binary
           # overriding CIRCLE_BUILD_NUM to avoid conflict with other tests
-          command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-dist garden-service/dist/linux-amd64/garden deploy --root examples/demo-project --env testing --logger-type basic
+          command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-dist garden-service/dist/linux-amd64/garden deploy --root examples/demo-project --env remote --logger-type basic
       - run:
           name: Cleanup
           command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-dist kubectl delete --wait=false $(kubectl get ns -o name | grep testing-$CIRCLE_BUILD_NUM) || true
@@ -398,10 +398,10 @@ jobs:
           command: $env:CIRCLE_BUILD_NUM = "$env:CIRCLE_BUILD_NUM-win"
       - run:
           name: Build demo-project
-          command: .\garden-service\dist\windows-amd64\garden.exe build --root .\examples\demo-project\ --logger-type basic --env testing
+          command: .\garden-service\dist\windows-amd64\garden.exe build --root .\examples\demo-project\ --logger-type basic --env remote
       - run:
           name: Deploy demo-project
-          command: .\garden-service\dist\windows-amd64\garden.exe deploy --root .\examples\demo-project\ --logger-type basic --env testing
+          command: .\garden-service\dist\windows-amd64\garden.exe deploy --root .\examples\demo-project\ --logger-type basic --env remote
       - run:
           name: Cleanup
           command: (kubectl delete namespace --wait=false demo-project-testing-$env:CIRCLE_BUILD_NUM demo-project-testing-$env:CIRCLE_BUILD_NUM--metadata) -or $true

examples/demo-project/garden.yml

@@ -10,15 +10,6 @@ environments:
       - name: kubernetes
         # Replace these values as appropriate
         context: gke_garden-dev-200012_europe-west1-b_garden-dev-1
-        namespace: ${local.env.USER || local.username}-demo-project
+        namespace: demo-project-testing-${local.env.USER || local.username}
         defaultHostname: ${local.env.USER || local.username}-demo-project.dev-1.sys.garden
         buildMode: cluster-docker
-  - name: testing
-    providers:
-      - name: kubernetes
-        context: gke_garden-dev-200012_europe-west1-b_garden-dev-1
-        namespace: demo-project-testing-${local.env.CIRCLE_BUILD_NUM || local.username}
-        defaultHostname: demo-project-testing.dev-1.sys.garden
-        buildMode: cluster-docker
-        clusterDocker:
-          enableBuildKit: true

examples/kaniko/README.md

@@ -0,0 +1,3 @@
+# kaniko
+
+A very basic demo project for Garden showing in-cluster building with [kaniko](https://github.com/GoogleContainerTools/kaniko).

examples/kaniko/backend/.dockerignore

@@ -0,0 +1,4 @@
+node_modules
+Dockerfile
+garden.yml
+app.yaml

examples/kaniko/backend/.gitignore

@@ -0,0 +1,27 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof
+
+.vscode/settings.json
+webserver/*server*

examples/kaniko/backend/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.8.3-alpine
+MAINTAINER Aurelien PERRIER <[email protected]>
+
+ENV webserver_path /go/src/github.com/perriea/webserver/
+ENV PATH $PATH:$webserver_path
+
+WORKDIR $webserver_path
+COPY webserver/ .
+
+RUN go build .
+
+ENTRYPOINT ./webserver
+
+EXPOSE 8080

examples/kaniko/backend/garden.yml

@@ -0,0 +1,14 @@
+kind: Module
+name: backend
+description: Backend service container
+type: container
+services:
+  - name: backend
+    ports:
+      - name: http
+        containerPort: 8080
+        # Maps service:80 -> container:8080
+        servicePort: 80
+    ingresses:
+      - path: /hello-backend
+        port: http

examples/kaniko/backend/webserver/main.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+)
+
+func handler(w http.ResponseWriter, r *http.Request) {
+	fmt.Fprint(w, "Hello from Go!")
+}
+
+func main() {
+	http.HandleFunc("/hello-backend", handler)
+	fmt.Println("Server running...")
+
+	http.ListenAndServe(":8080", nil)
+}

examples/kaniko/frontend/.dockerignore

@@ -0,0 +1,4 @@
+node_modules
+Dockerfile
+garden.yml
+app.yaml

examples/kaniko/frontend/Dockerfile

@@ -0,0 +1,12 @@
+FROM node:9-alpine
+
+ENV PORT=8080
+EXPOSE ${PORT}
+WORKDIR /app
+
+ADD package.json /app
+RUN npm install
+
+ADD . /app
+
+CMD ["npm", "start"]

examples/kaniko/frontend/app.js

@@ -0,0 +1,27 @@
+const express = require('express');
+const request = require('request-promise')
+const app = express();
+
+const backendServiceEndpoint = `http://backend/hello-backend`
+
+app.get('/hello-frontend', (req, res) => res.send('Hello from the frontend!'));
+
+app.get('/call-backend', (req, res) => {
+  // Query the backend and return the response
+  request.get(backendServiceEndpoint)
+    .then(message => {
+      message = `Backend says: '${message}'`
+      res.json({
+        message,
+      })
+    })
+    .catch(err => {
+      res.statusCode = 500
+      res.json({
+        error: err,
+        message: "Unable to reach service at " + backendServiceEndpoint,
+      })
+    });
+});
+
+module.exports = { app }

examples/kaniko/frontend/garden.yml

@@ -0,0 +1,27 @@
+kind: Module
+name: frontend
+description: Frontend service container
+type: container
+services:
+  - name: frontend
+    ports:
+      - name: http
+        containerPort: 8080
+    healthCheck:
+      httpGet:
+        path: /hello-frontend
+        port: http
+    ingresses:
+      - path: /hello-frontend
+        port: http
+      - path: /call-backend
+        port: http
+    dependencies:
+      - backend
+tests:
+  - name: unit
+    args: [npm, test]
+  - name: integ
+    args: [npm, run, integ]
+    dependencies:
+      - frontend

examples/kaniko/frontend/main.js

@@ -0,0 +1,3 @@
+const { app } = require('./app');
+
+app.listen(process.env.PORT, '0.0.0.0', () => console.log('Frontend service started'));

examples/kaniko/frontend/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "frontend",
+  "version": "1.0.0",
+  "description": "Simple Node.js docker service",
+  "main": "main.js",
+  "scripts": {
+    "start": "node main.js",
+    "test": "echo OK",
+    "integ": "node_modules/mocha/bin/mocha test/integ.js"
+  },
+  "author": "garden.io <[email protected]>",
+  "license": "ISC",
+  "dependencies": {
+    "express": "^4.16.2",
+    "request": "^2.83.0",
+    "request-promise": "^4.2.2"
+  },
+  "devDependencies": {
+    "mocha": "^5.1.1",
+    "supertest": "^3.0.0"
+  }
+}

examples/kaniko/frontend/test/integ.js

@@ -0,0 +1,17 @@
+const supertest = require("supertest")
+const { app } = require("../app")
+
+describe('GET /call-backend', () => {
+  const agent = supertest.agent(app)
+
+  it('should respond with a message from the backend service', (done) => {
+    agent
+      .get("/call-backend")
+      .expect(200, { message: "Backend says: 'Hello from Go!'" })
+      .end((err) => {
+        if (err) return done(err)
+        done()
+      })
+  })
+})
+

examples/kaniko/garden.yml

@@ -0,0 +1,15 @@
+kind: Project
+name: kaniko
+# defaultEnvironment: "remote" # Uncomment if you'd like the remote environment to be the default for this project.
+environments:
+  - name: local
+    providers:
+      - name: local-kubernetes
+  - name: remote
+    providers:
+      - name: kubernetes
+        # Replace these values as appropriate
+        context: gke_garden-dev-200012_europe-west1-b_garden-dev-1
+        namespace: ${local.env.USER || local.username}-demo-project
+        defaultHostname: ${local.env.USER || local.username}-demo-project.dev-1.sys.garden
+        buildMode: kaniko

garden-service/src/plugins/kubernetes/container/build.ts

@@ -23,7 +23,7 @@ import { LogEntry } from "../../../logger/log-entry"
 import { KubernetesProvider, ContainerBuildMode, KubernetesPluginContext } from "../config"
 import { PluginError } from "../../../exceptions"
 import { PodRunner } from "../run"
-import { getRegistryHostname } from "../init"
+import { getRegistryHostname, getKubernetesSystemVariables } from "../init"
 import { getManifestFromRegistry } from "./util"
 import { normalizeLocalRsyncPath } from "../../../util/fs"
 import { getPortForward } from "../port-forward"
@@ -36,7 +36,7 @@ const dockerDaemonContainerName = "docker-daemon"
 // Note: v0.9.0 appears to be completely broken: https://github.com/GoogleContainerTools/kaniko/issues/268
 const kanikoImage = "gcr.io/kaniko-project/executor:v0.8.0"
 const registryPort = 5000
-const syncDataVolumeName = "garden-build-sync"
+
 export const buildSyncDeploymentName = "garden-build-sync"
 
 export async function k8sGetContainerBuildStatus(params: GetBuildStatusParams<ContainerModule>): Promise<BuildStatus> {
@@ -313,6 +313,8 @@ async function runKaniko({ provider, log, module, args, outputStream }: RunKanik
   const api = await KubeApi.factory(log, provider)
   const podName = `kaniko-${module.name}-${Math.round(new Date().getTime())}`
   const registryHostname = getRegistryHostname()
+  const k8sSystemVars = getKubernetesSystemVariables(provider.config)
+  const syncDataVolumeName = k8sSystemVars["sync-volume-name"]
 
   const runner = new PodRunner({
     api,

garden-service/src/plugins/kubernetes/system.ts

@@ -233,7 +233,7 @@ export async function prepareSystemServices({
     if (failed.length === 1) {
       const error = errors[0]
 
-      throw new PluginError(`${provider.name}—an error occurred when configuring environment:\n${error}`, {
+      throw new PluginError(`${provider.name} — an error occurred when configuring environment:\n${error}`, {
         error,
         results,
       })

garden-service/test/e2e/garden.yml

@@ -26,7 +26,10 @@ name: e2e-tests
 type: exec
 tests:
   - name: demo-project
-    command: [npm, run, e2e-project, --, --project=demo-project, "--showlog=${var.show-log}", "--env=${environment.name}"]
+    command: [npm, run, e2e-project, --, --project=demo-project, "--showlog=${var.show-log}", "--env=remote"]
+    timeout: ${var.timeout}
+  - name: kaniko
+    command: [npm, run, e2e-project, --, --project=kaniko, "--showlog=${var.show-log}", "--env=remote"]
     timeout: ${var.timeout}
   - name: tasks # Tests for tasks are currently being skipped
     command: [npm, run, e2e-project, --, --project=tasks, "--showlog=${var.show-log}", "--env=${environment.name}"]