ci: run integ tests with Garden in CI (#842)

* ci: run integ tests with Garden in CI

.circleci/config.yml

@@ -130,6 +130,23 @@ commands:
             docker tag << parameters.context >>  << parameters.registry >>/<< parameters.image >>:$TAG
             docker push << parameters.registry >>/<< parameters.image >>:$TAG
 
+  configure_kubectl_context:
+    description: Configure the kubectl context so that we can access our remote cluster. Used for integration testing.
+    steps:
+      - run:
+          name: Install gcloud
+          command: |
+            mkdir $HOME/gcloud
+            curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz | tar xvz -C $HOME/gcloud
+            $HOME/gcloud/google-cloud-sdk/install.sh --quiet
+            echo 'export PATH=$HOME/gcloud/google-cloud-sdk/bin:$PATH' >> $BASH_ENV
+      - run:
+          name: Configure kubectl context via gcloud and authenticate to Google Container Registry
+          command: |
+            echo $GCLOUD_SERVICE_KEY | gcloud auth activate-service-account --key-file=-
+            gcloud --quiet config set project $GOOGLE_PROJECT_ID && gcloud --quiet config set compute/zone $GOOGLE_COMPUTE_ZONE
+            gcloud --quiet container clusters get-credentials $GOOGLE_CLUSTER_ID --zone $GOOGLE_COMPUTE_ZONE
+            gcloud --quiet auth configure-docker
 #
 # Jobs section
 #
@@ -182,6 +199,20 @@ jobs:
             npm test
           environment:
             CHOKIDAR_USEPOLLING: "1"
+  integ-test-service:
+    <<: *node-config
+    steps:
+      - checkout
+      - run: sudo apt-get update && sudo apt-get install rsync
+      - npm_install
+      - configure_kubectl_context
+      - *attach-workspace
+      - run:
+          name: Init Garden
+          command: ./garden-service/bin/garden init --root garden-service/test/integ --logger-type basic
+      - run:
+          name: Run integration tests
+          command: ./garden-service/bin/garden test integ-tests --root garden-service/test/integ --logger-type basic
   build-dashboard:
     <<: *node-config
     steps:
@@ -253,6 +284,9 @@ workflows:
       - test-service:
           requires:
             - build-service
+      - integ-test-service:
+          requires:
+            - build-service
       - build-dashboard
       - test-dashboard:
           requires:

examples/demo-project/garden.yml

@@ -11,4 +11,10 @@ environments:
         context: gke_garden-dev-200012_europe-west1-b_garden-dev-1
         namespace: ${local.env.USER || "default"}-demo-project
         defaultHostname: ${local.env.USER || "default"}-demo-project.dev-1.sys.garden
+  - name: testing
+    providers:
+      - name: kubernetes
+        context: gke_garden-dev-200012_europe-west1-b_garden-dev-1
+        namespace: demo-project-testing-${local.env.CIRCLE_BUILD_NUM || "default"}
+        defaultHostname: demo-project-testing.dev-1.sys.garden
         buildMode: cluster-docker

examples/hot-reload/garden.yml

@@ -7,3 +7,10 @@ environments:
   - name: local
     providers:
       - name: local-kubernetes
+  - name: testing
+    providers:
+      - name: kubernetes
+        context: gke_garden-dev-200012_europe-west1-b_garden-dev-1
+        namespace: hot-reload-testing-${local.env.CIRCLE_BUILD_NUM || "default"}
+        defaultHostname: hot-reload-testing.dev-1.sys.garden
+        buildMode: cluster-docker

examples/remote-sources/garden.yml

@@ -5,3 +5,11 @@ sources:
     repositoryUrl: https://github.com/garden-io/garden-example-remote-sources-web-services.git#v0.1.0
   - name: db-services
     repositoryUrl: https://github.com/garden-io/garden-example-remote-sources-db-services.git#v0.1.0
+environments:
+  - name: testing
+    providers:
+      - name: kubernetes
+        context: gke_garden-dev-200012_europe-west1-b_garden-dev-1
+        namespace: remote-sources-testing-${local.env.CIRCLE_BUILD_NUM || "default"}
+        defaultHostname: remote-sources-testing.dev-1.sys.garden
+        buildMode: cluster-docker

examples/tasks/garden.yml

@@ -4,3 +4,10 @@ environments:
   - name: local
     providers:
       - name: local-kubernetes
+  - name: testing
+    providers:
+      - name: kubernetes
+        context: gke_garden-dev-200012_europe-west1-b_garden-dev-1
+        namespace: tasks-testing-${local.env.CIRCLE_BUILD_NUM || "default"}
+        defaultHostname: tasks-testing.dev-1.sys.garden
+        buildMode: cluster-docker

examples/vote-helm/garden.yml

@@ -4,3 +4,10 @@ environments:
   - name: local
     providers:
       - name: local-kubernetes
+  - name: testing
+    providers:
+      - name: kubernetes
+        context: gke_garden-dev-200012_europe-west1-b_garden-dev-1
+        namespace: vote-helm-testing-${local.env.CIRCLE_BUILD_NUM || "default"}
+        defaultHostname: vote-helm-testing.dev-1.sys.garden
+        buildMode: cluster-docker

examples/vote/garden.yml

@@ -11,4 +11,11 @@ environments:
         context: gke_garden-dev-200012_europe-west1-b_garden-dev-1
         namespace: ${local.env.USER || "default"}-vote
         defaultHostname: ${local.env.USER || "default"}-vote.dev-1.sys.garden
-        buildMode: cluster-docker
+        buildMode: cluster-docker
+  - name: testing
+    providers:
+      - name: kubernetes
+        context: gke_garden-dev-200012_europe-west1-b_garden-dev-1
+        namespace: vote-testing-${local.env.CIRCLE_BUILD_NUM || "default"}
+        defaultHostname: vote-testing.dev-1.sys.garden
+        buildMode: cluster-docker

garden-service/package.json

@@ -191,7 +191,7 @@
     "dev": "npm link && npm run tsc",
     "dist": "npm run build",
     "integ": "./test/integ/run",
-    "integ-full": "./build/test/integ/integ-full.js",
+    "integ-full": "node ./build/test/integ/integ-full.js",
     "prepare": "npm run snyk-protect",
     "snyk-protect": "snyk protect",
     "test": "node_modules/.bin/mocha",

garden-service/test/integ-helpers.ts

@@ -1,5 +1,6 @@
 import * as execa from "execa"
 import * as Bluebird from "bluebird"
+import * as mlog from "mocha-logger"
 import { remove } from "fs-extra"
 import { get, intersection } from "lodash"
 import parseArgs = require("minimist")
@@ -74,7 +75,13 @@ export async function touchFile(path: string): Promise<void> {
 
 export function parseLogEntries(entries: string[]): JsonLogEntry[] {
   return entries.filter(Boolean).map((line) => {
-    return JSON.parse(line)
+    // Lines are not always JSON parseable
+    try {
+      return JSON.parse(line)
+    } catch (error) {
+      mlog.log("Unable to parse line", line)
+      return {}
+    }
   })
 }
 

garden-service/test/integ/README.md

@@ -11,7 +11,7 @@ The script performs the following cleanup operations before running the tests:
 The `integ-full` script supports the following options:
 
 * `binPath`: The Garden binary to use for the tests (defaults to the one in the static directory). Useful for testing release binaries.
-* `only`: Runs only the test sequence specified (e.g. `simple-project` or `vote-helm`).
+* `only`: Runs only the test sequence specified (e.g. `demo-project` or `vote-helm`).
 
 For example:
 ```

garden-service/test/integ/garden.yml

@@ -1,25 +1,29 @@
 kind: Project
 name: integ-tests
 environments:
-  - name: local
+  - name: testing
     providers:
-      - name: local-kubernetes
-  # TODO
-  # - name: remote
-  #   providers:
-  #     - name: kubernetes
+      - name: kubernetes
+        context: gke_garden-dev-200012_europe-west1-b_garden-dev-1
+        namespace: integ-tests
+        defaultHostname: dev-1.sys.garden
+        buildMode: cluster-docker
 
 ---
 
 kind: Module
 name: integ-tests
 type: exec
 tests:
-  - name: simple-project
-    command: [npm, run, integ-full, --, --only=simple-project, "--showlog=true"]
-  - name: tasks
-    command: [npm, run, integ-full, --, --only=tasks, "--showlog=true"]
+  - name: demo-project
+    command: [npm, run, integ-full, --, --only=demo-project, --showlog=true, --env=testing]
+  - name: tasks # Tests for tasks are currently being skipped
+    command: [npm, run, integ-full, --, --only=tasks, --showlog=true, --env=testing]
+  - name: hot-reload # Tests for hot-reload are currently being skipped
+    command: [npm, run, integ-full, --, --only=hot-reload, --showlog=true, --env=testing]
   - name: vote-helm
-    command: [npm, run, integ-full, --, --only=vote-helm, "--showlog=true"]
+    command: [npm, run, integ-full, --, --only=vote-helm, --showlog=true, --env=testing]
+  - name: vote
+    command: [npm, run, integ-full, --, --only=vote, --showlog=true, --env=testing]
   - name: remote-sources
-    command: [npm, run, integ-full, --, --only=remote-sources, "--showlog=true"]
+    command: [npm, run, integ-full, --, --only=remote-sources, --showlog=true, --env=testing]

garden-service/test/integ/integ-full.ts

@@ -1,14 +1,11 @@
 import * as execa from "execa"
-import * as Bluebird from "bluebird"
 import parseArgs = require("minimist")
 import { resolve } from "path"
-import {
-  deleteExampleNamespaces,
-  deleteSystemMetadataNamespace,
-} from "../integ-helpers"
 import { examplesDir } from "../helpers"
 import { dedent } from "../../src/util/string"
 import chalk from "chalk"
+import { InternalError } from "../../src/exceptions"
+import { GARDEN_SERVICE_ROOT } from "../../src/constants"
 
 export const parsedArgs = parseArgs(process.argv.slice(2))
 
@@ -20,13 +17,22 @@ ${chalk.green("-h")}: Prints this message and quits.
 ${chalk.green("--binPath")}: Uses the garden binary at the path provided instead \
 of the one at ${chalk.blue("[garden-root]/bin/garden")}.
 
+${chalk.green("--env")}: The environment to run the test in. \
+E.g. ${chalk.blue("local")} or ${chalk.blue("testing")}.
+
 ${chalk.green("--only")}: Runs only the test sequence indicated. \
 E.g. ${chalk.blue("demo-project")} or ${chalk.blue("vote-helm")}.
 
 Example: ./garden-service/bin/integ-full.ts --binPath=/path/to/garden --only=demo-project
 `
 
 async function run() {
+  // Abort if examples dir is dirty to prevent changes being checked out
+  try {
+    await execa("git", ["diff-index", "--quiet", "HEAD", examplesDir])
+  } catch (_error) {
+    throw new InternalError("Examples directory is dirty. Aborting.", {})
+  }
 
   if (parsedArgs["h"]) {
     console.log(usageStr)
@@ -35,29 +41,19 @@ async function run() {
 
   console.log(chalk.grey("Call this script with -h for usage information."))
   console.log("Starting integ tests.")
-  console.log("Clearing example namespaces and system metadata namespace...")
-
-  await Bluebird.all([deleteSystemMetadataNamespace(), deleteExampleNamespaces()])
-
-  const gardenServiceRoot = resolve(__dirname, "../")
-
-  console.log("Checking out examples dir...")
-  await execa("git", ["checkout", examplesDir])
 
   console.log("Running tests...")
 
   const mochaOpts = ["--opts", "test/mocha.integ.opts"]
 
-  for (const opt of ["binPath", "only"]) {
+  for (const opt of ["binPath", "only", "env"]) {
     if (parsedArgs[opt]) {
       mochaOpts.push(`--${opt}`, parsedArgs[opt])
     }
   }
 
-  const mochaBinPath = resolve(gardenServiceRoot, "node_modules/.bin/mocha")
-  await execa(mochaBinPath, mochaOpts, { cwd: gardenServiceRoot, stdio: "inherit" })
-  console.log("Checking out examples dir...")
-  await execa("git", ["checkout", examplesDir])
+  const mochaBinPath = resolve(GARDEN_SERVICE_ROOT, "node_modules/.bin/mocha")
+  await execa(mochaBinPath, mochaOpts, { cwd: GARDEN_SERVICE_ROOT, stdio: "inherit" })
   console.log("Done.")
 }