fix: delete outdated system namespaces

* Add a version annotation to k8s namespaces created by the framework.

* Introduced a minimum compatible version constraint for the
garden-system / garden-system--metadata k8s namespaces. If these
namespaces were created by a Garden version older than the current
minimum compatible version, they'll be deleted and recreated.

* Added a releaseName to the k8s ingress-controller system module.

garden-service/src/actions.ts

@@ -181,8 +181,10 @@ export class ActionHelper implements TypeGuard {
     // sequentially go through the preparation steps, to allow plugins to request user input
     for (const [name, handler] of Object.entries(handlers)) {
       const status = statuses[name] || { ready: false }
+      const needForce = status.detail && !!status.detail.needForce
+      const forcePrep = force || needForce
 
-      if (status.ready && !force) {
+      if (status.ready && !forcePrep) {
         continue
       }
 
@@ -192,7 +194,7 @@ export class ActionHelper implements TypeGuard {
         msg: "Preparing environment...",
       })
 
-      await handler({ ...this.commonParams(handler, log), force, status, log: envLogEntry })
+      await handler({ ...this.commonParams(handler, log), force: forcePrep, status, log: envLogEntry })
 
       envLogEntry.setSuccess("Configured")
 

garden-service/src/cli/cli.ts

@@ -12,7 +12,7 @@ import { resolve } from "path"
 import { safeDump } from "js-yaml"
 import { coreCommands } from "../commands/commands"
 import { DeepPrimitiveMap } from "../config/common"
-import { shutdown, sleep } from "../util/util"
+import { shutdown, sleep, getPackageVersion } from "../util/util"
 import {
   BooleanParameter,
   ChoicesParameter,
@@ -42,7 +42,6 @@ import {
   prepareArgConfig,
   prepareOptionConfig,
   styleConfig,
-  getPackageVersion,
   getLogLevelChoices,
   parseLogLevel,
 } from "./helpers"

garden-service/src/cli/helpers.ts

@@ -199,8 +199,3 @@ export function failOnInvalidOptions(argv, ctx) {
     ctx.cliMessage(`Received invalid flag(s): ${invalid.join(", ")}`)
   }
 }
-
-export function getPackageVersion(): String {
-  const version = require("../../package.json").version
-  return version
-}

garden-service/src/commands/version.ts

@@ -11,8 +11,8 @@ import {
   CommandResult,
   CommandParams,
 } from "./base"
-import { getPackageVersion } from "../cli/helpers"
 import chalk from "chalk"
+import { getPackageVersion } from "../util/util"
 
 export class VersionCommand extends Command {
   name = "version"

garden-service/src/plugins/kubernetes/init.ts

@@ -9,7 +9,8 @@
 import * as Bluebird from "bluebird"
 import * as inquirer from "inquirer"
 import * as Joi from "joi"
-import { uniq, every, values, pick, find } from "lodash"
+import * as semver from "semver"
+import { every, find, intersection, pick, uniq, values } from "lodash"
 
 import { DeploymentError, NotFoundError, TimeoutError, PluginError } from "../../exceptions"
 import {
@@ -18,13 +19,15 @@ import {
   GetEnvironmentStatusParams,
   PluginActionParamsBase,
 } from "../../types/plugin/params"
-import { sleep } from "../../util/util"
+import { deline } from "../../util/string"
+import { sleep, getPackageVersion } from "../../util/util"
 import { joiUserIdentifier } from "../../config/common"
 import { KubeApi } from "./api"
 import {
   getAppNamespace,
   getMetadataNamespace,
   getAllNamespaces,
+  createNamespace,
 } from "./namespace"
 import { KUBECTL_DEFAULT_TIMEOUT, kubectl } from "./kubectl"
 import { name as providerName, KubernetesProvider } from "./kubernetes"
@@ -35,6 +38,8 @@ import { DashboardPage } from "../../config/dashboard"
 import { checkTillerStatus, installTiller } from "./helm/tiller"
 
 const MAX_STORED_USERNAMES = 5
+const GARDEN_VERSION = getPackageVersion()
+const SYSTEM_NAMESPACE_MIN_VERSION = "0.9.0"
 
 /**
  * Used by both the remote and local plugin
@@ -78,17 +83,26 @@ export async function getRemoteEnvironmentStatus({ ctx, log }: GetEnvironmentSta
 
   await prepareNamespaces({ ctx, log })
 
-  const ready = (await checkTillerStatus(ctx, ctx.provider, log)) === "ready"
+  let ready = (await checkTillerStatus(ctx, ctx.provider, log)) === "ready"
+
+  const api = new KubeApi(ctx.provider)
+  const contextForLog = `Checking environment status for plugin "kubernetes"`
+  const sysNamespaceUpToDate = await systemNamespaceUpToDate(api, log, contextForLog)
+  if (!sysNamespaceUpToDate) {
+    ready = false
+  }
 
   return {
     ready,
     needUserInput: false,
+    detail: { needForce: !sysNamespaceUpToDate },
   }
 }
 
 export async function getLocalEnvironmentStatus({ ctx, log }: GetEnvironmentStatusParams) {
   let ready = true
   let needUserInput = false
+  let sysNamespaceUpToDate = true
   const dashboardPages: DashboardPage[] = []
 
   await prepareNamespaces({ ctx, log })
@@ -101,8 +115,14 @@ export async function getLocalEnvironmentStatus({ ctx, log }: GetEnvironmentStat
 
     const serviceStatuses = pick(sysStatus.services, getSystemServices(ctx.provider))
 
+    const api = new KubeApi(ctx.provider)
+
     const servicesReady = every(values(serviceStatuses).map(s => s.state === "ready"))
-    const systemReady = sysStatus.providers[ctx.provider.config.name].ready && servicesReady
+    const contextForLog = `Checking environment status for plugin "local-kubernetes"`
+    sysNamespaceUpToDate = await systemNamespaceUpToDate(api, log, contextForLog)
+    const systemReady = sysStatus.providers[ctx.provider.config.name].ready
+      && servicesReady
+      && sysNamespaceUpToDate
 
     if (!systemReady) {
       ready = false
@@ -145,6 +165,7 @@ export async function getLocalEnvironmentStatus({ ctx, log }: GetEnvironmentStat
     ready,
     needUserInput,
     dashboardPages,
+    detail: { needForce: !sysNamespaceUpToDate },
   }
 }
 
@@ -155,6 +176,11 @@ export async function prepareRemoteEnvironment({ ctx, log }: PrepareEnvironmentP
     await login({ ctx, log })
   }
 
+  const api = new KubeApi(ctx.provider)
+  const contextForLog = `Preparing environment for plugin "kubernetes"`
+  if (!await systemNamespaceUpToDate(api, log, contextForLog)) {
+    await recreateSystemNamespaces(api, log)
+  }
   await installTiller(ctx, ctx.provider, log)
 
   return {}
@@ -163,13 +189,63 @@ export async function prepareRemoteEnvironment({ ctx, log }: PrepareEnvironmentP
 export async function prepareLocalEnvironment({ ctx, force, log }: PrepareEnvironmentParams) {
   // make sure system services are deployed
   if (!isSystemGarden(ctx.provider)) {
-    await configureSystemServices({ ctx, force, log })
+    const api = new KubeApi(ctx.provider)
+    const contextForLog = `Preparing environment for plugin "local-kubernetes"`
+    const outdated = !(await systemNamespaceUpToDate(api, log, contextForLog))
+    if (outdated) {
+      await recreateSystemNamespaces(api, log)
+    }
+    await configureSystemServices({ ctx, log, force: force || outdated })
     await installTiller(ctx, ctx.provider, log)
   }
 
   return {}
 }
 
+/**
+ * Returns true if the garden-system namespace exists and has the version
+ */
+export async function systemNamespaceUpToDate(api: KubeApi, log: LogEntry, contextForLog: string): Promise<boolean> {
+  let systemNamespace
+  try {
+    systemNamespace = await api.core.readNamespace("garden-system")
+  } catch (err) {
+    if (err.code === 404) {
+      return false
+    } else {
+      throw err
+    }
+  }
+
+  const versionInCluster = systemNamespace.body.metadata.annotations["garden.io/version"]
+
+  const upToDate = !!versionInCluster && semver.gte(semver.coerce(versionInCluster)!, SYSTEM_NAMESPACE_MIN_VERSION)
+
+  log.debug(deline`
+    ${contextForLog}: current version ${GARDEN_VERSION}, version in cluster: ${versionInCluster},
+    oldest permitted version: ${SYSTEM_NAMESPACE_MIN_VERSION}, up to date: ${upToDate}
+  `)
+
+  return upToDate
+}
+
+/**
+ * Returns true if the garden-system namespace was outdated.
+ */
+export async function recreateSystemNamespaces(api: KubeApi, log: LogEntry) {
+  const entry = log.debug({
+    section: "cleanup",
+    msg: "Deleting outdated system namespaces...",
+    status: "active",
+  })
+  await deleteNamespaces(["garden-system", "garden-system--metadata"], api, log)
+  entry.setState({ msg: "Creating system namespaces..." })
+  await createNamespace(api, "garden-system")
+  await createNamespace(api, "garden-system--metadata")
+  entry.setState({ msg: "System namespaces up to date" })
+  entry.setSuccess()
+}
+
 export async function cleanupEnvironment({ ctx, log }: CleanupEnvironmentParams) {
   const api = new KubeApi(ctx.provider)
   const namespace = await getAppNamespace(ctx, ctx.provider)
@@ -179,39 +255,44 @@ export async function cleanupEnvironment({ ctx, log }: CleanupEnvironmentParams)
     status: "active",
   })
 
-  try {
-    // Note: Need to call the delete method with an empty object
-    // TODO: any cast is required until https://github.com/kubernetes-client/javascript/issues/52 is fixed
-    await api.core.deleteNamespace(namespace, <any>{})
-  } catch (err) {
-    entry.setError(err.message)
-    const availableNamespaces = await getAllNamespaces(api)
-    throw new NotFoundError(err, { namespace, availableNamespaces })
-  }
-
+  await deleteNamespaces([namespace], api, entry)
   await logout({ ctx, log })
 
+  return {}
+}
+
+export async function deleteNamespaces(namespaces: string[], api: KubeApi, log: LogEntry) {
+  for (const ns of namespaces) {
+    try {
+      // Note: Need to call the delete method with an empty object
+      // TODO: any cast is required until https://github.com/kubernetes-client/javascript/issues/52 is fixed
+      await api.core.deleteNamespace(ns, <any>{})
+    } catch (err) {
+      log.setError(err.message)
+      const availableNamespaces = await getAllNamespaces(api)
+      throw new NotFoundError(err, { namespace: ns, availableNamespaces })
+    }
+  }
+
   // Wait until namespace has been deleted
   const startTime = new Date().getTime()
   while (true) {
     await sleep(2000)
 
     const nsNames = await getAllNamespaces(api)
-    if (!nsNames.includes(namespace)) {
-      entry.setSuccess()
+    if (intersection(nsNames, namespaces).length === 0) {
+      log.setSuccess()
       break
     }
 
     const now = new Date().getTime()
     if (now - startTime > KUBECTL_DEFAULT_TIMEOUT * 1000) {
       throw new TimeoutError(
-        `Timed out waiting for namespace ${namespace} delete to complete`,
-        { namespace },
+        `Timed out waiting for namespace ${namespaces.join(", ")} delete to complete`,
+        { namespaces },
       )
     }
   }
-
-  return {}
 }
 
 async function getLoginStatus({ ctx }: PluginActionParamsBase) {

garden-service/src/plugins/kubernetes/namespace.ts

@@ -11,7 +11,9 @@ import { KubeApi } from "./api"
 import { KubernetesProvider } from "./kubernetes"
 import { name as providerName } from "./kubernetes"
 import { AuthenticationError } from "../../exceptions"
+import { getPackageVersion } from "../../util/util"
 
+const GARDEN_VERSION = getPackageVersion()
 const created: { [name: string]: boolean } = {}
 
 export async function ensureNamespace(api: KubeApi, namespace: string) {
@@ -26,21 +28,28 @@ export async function ensureNamespace(api: KubeApi, namespace: string) {
 
     if (!created[namespace]) {
       // TODO: the types for all the create functions in the library are currently broken
-      await api.core.createNamespace(<any>{
-        apiVersion: "v1",
-        kind: "Namespace",
-        metadata: {
-          name: namespace,
-          annotations: {
-            "garden.io/generated": "true",
-          },
-        },
-      })
+      await createNamespace(api, namespace)
       created[namespace] = true
     }
   }
 }
 
+// Note: Does not check whether the namespace already exists.
+export async function createNamespace(api: KubeApi, namespace: string) {
+  // TODO: the types for all the create functions in the library are currently broken
+  return api.core.createNamespace(<any>{
+    apiVersion: "v1",
+    kind: "Namespace",
+    metadata: {
+      name: namespace,
+      annotations: {
+        "garden.io/generated": "true",
+        "garden.io/version": GARDEN_VERSION,
+      },
+    },
+  })
+}
+
 export async function getNamespace(
   { ctx, provider, suffix, skipCreate }:
     { ctx: PluginContext, provider: KubernetesProvider, suffix?: string, skipCreate?: boolean },

garden-service/src/util/util.ts

@@ -61,6 +61,11 @@ export function registerCleanupFunction(name: string, func: HookCallback) {
   exitHook(func)
 }
 
+export function getPackageVersion(): String {
+  const version = require("../../package.json").version
+  return version
+}
+
 /*
   Warning: Don't make any async calls in the loop body when using this function, since this may cause
   funky concurrency behavior.

garden-service/static/kubernetes/system/ingress-controller/garden.yml

@@ -3,6 +3,7 @@ module:
   name: ingress-controller
   type: helm
   chart: stable/nginx-ingress
+  releaseName: garden-nginx
   dependencies:
     - default-backend
   version: 0.25.1

garden-service/test/src/cli/helpers.ts

@@ -7,8 +7,9 @@
  */
 
 import { expect } from "chai"
-import { getPackageVersion, parseLogLevel, getLogLevelChoices } from "../../../src/cli/helpers"
+import { parseLogLevel, getLogLevelChoices } from "../../../src/cli/helpers"
 import { expectError } from "../../helpers"
+import { getPackageVersion } from "../../../src/util/util"
 
 describe("helpers", () => {
   const validLogLevels = ["error", "warn", "info", "verbose", "debug", "silly", "0", "1", "2", "3", "4", "5"]