refactor(k8s): allow overriding the default garden-system namespace

Flagging as a refactor because this is mainly intended for internal
usage for now, since changing the namespace might have some unintended
consequences.

garden-service/src/garden.ts

@@ -143,6 +143,7 @@ export class Garden {
   public readonly moduleIncludePatterns?: string[]
   public readonly moduleExcludePatterns: string[]
   public readonly persistent: boolean
+  public readonly systemNamespace: string
 
   constructor(params: GardenParams) {
     this.buildDir = params.buildDir

garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts

@@ -20,7 +20,6 @@ import { splitFirst, splitLast } from "../../../util/util"
 import { LogEntry } from "../../../logger/log-entry"
 import Bluebird from "bluebird"
 import { CLUSTER_REGISTRY_DEPLOYMENT_NAME } from "../constants"
-import { systemNamespace } from "../system"
 import { PluginError } from "../../../exceptions"
 import { apply, kubectl } from "../kubectl"
 import { waitForResources } from "../status/status"
@@ -199,6 +198,7 @@ async function runRegistryGarbageCollection(ctx: KubernetesPluginContext, api: K
   })
 
   const provider = ctx.provider
+  const systemNamespace = provider.config.gardenSystemNamespace
 
   // Restart the registry in read-only mode
   // -> Get the original deployment
@@ -404,6 +404,7 @@ async function cleanupBuildSyncVolume(provider: KubernetesProvider, log: LogEntr
 // (doesn't matter which one, they all use the same volume)
 async function getBuildSyncPodName(provider: KubernetesProvider, log: LogEntry) {
   const api = await KubeApi.factory(log, provider)
+  const systemNamespace = provider.config.gardenSystemNamespace
 
   const builderStatusRes = await api.apps.readNamespacedDeployment(buildSyncDeploymentName, systemNamespace)
   const builderPods = await getPods(api, systemNamespace, builderStatusRes.spec.selector.matchLabels)
@@ -421,6 +422,7 @@ async function getBuildSyncPodName(provider: KubernetesProvider, log: LogEntry)
 
 async function execInBuildSync({ provider, log, args, timeout, podName }: BuilderExecParams) {
   const execCmd = ["exec", "-i", podName, "--", ...args]
+  const systemNamespace = provider.config.gardenSystemNamespace
 
   log.verbose(`Running: kubectl ${execCmd.join(" ")}`)
 

garden-service/src/plugins/kubernetes/commands/cluster-init.ts

@@ -10,7 +10,7 @@ import { PluginCommand } from "../../../types/plugin/command"
 import { prepareSystem, getEnvironmentStatus } from "../init"
 import chalk from "chalk"
 import { helm } from "../helm/helm-cli"
-import { KubernetesPluginContext } from "../config"
+import { KubernetesPluginContext, KubernetesProvider } from "../config"
 
 export const clusterInit: PluginCommand = {
   name: "cluster-init",
@@ -21,6 +21,7 @@ export const clusterInit: PluginCommand = {
   },
 
   handler: async ({ ctx, log }) => {
+    const provider = ctx.provider as KubernetesProvider
     const status = await getEnvironmentStatus({ ctx, log })
     let result = {}
 
@@ -44,7 +45,7 @@ export const clusterInit: PluginCommand = {
       await helm({
         ctx: k8sCtx,
         log,
-        namespace: "garden-system",
+        namespace: provider.config.gardenSystemNamespace,
         args: ["delete", "--purge", "garden-nfs-provisioner"],
       })
     } catch (_) {}

garden-service/src/plugins/kubernetes/config.ts

@@ -13,6 +13,7 @@ import { Provider, providerConfigBaseSchema, ProviderConfig } from "../../config
 import { containerRegistryConfigSchema, ContainerRegistryConfig } from "../container/config"
 import { PluginContext } from "../../plugin-context"
 import { deline } from "../../util/string"
+import { defaultSystemNamespace } from "./system"
 
 export interface ProviderSecretRef {
   name: string
@@ -100,6 +101,7 @@ export interface KubernetesBaseConfig extends ProviderConfig {
   registryProxyTolerations: Toleration[]
   resources: KubernetesResources
   storage: KubernetesStorage
+  gardenSystemNamespace: string
   tlsCertificates: IngressTlsCertificate[]
   certManager?: CertManagerConfig
   _systemServices: string[]
@@ -345,6 +347,16 @@ export const kubernetesConfigBase = providerConfigBaseSchema.keys({
       "Require SSL on all `container` module services. If set to true, an error is raised when no certificate " +
         "is available for a configured hostname on a `container` module."
     ),
+  gardenSystemNamespace: joi
+    .string()
+    .default(defaultSystemNamespace)
+    .description(
+      dedent`
+      Override the garden-system namespace name. This option is mainly used for testing.
+      In most cases you should leave the default value.
+      `
+    )
+    .meta({ internal: true }),
   imagePullSecrets: imagePullSecretsSchema,
   // TODO: invert the resources and storage config schemas
   resources: joi

garden-service/src/plugins/kubernetes/container/build.ts

@@ -14,7 +14,6 @@ import { buildContainerModule, getContainerBuildStatus, getDockerBuildFlags } fr
 import { GetBuildStatusParams, BuildStatus } from "../../../types/plugin/module/getBuildStatus"
 import { BuildModuleParams, BuildResult } from "../../../types/plugin/module/build"
 import { millicpuToString, megabytesToString, getRunningPodInDeployment } from "../util"
-import { systemNamespace } from "../system"
 import { RSYNC_PORT } from "../constants"
 import { posix, resolve } from "path"
 import { KubeApi } from "../api"
@@ -119,6 +118,7 @@ const localBuild: BuildHandler = async (params) => {
 const remoteBuild: BuildHandler = async (params) => {
   const { ctx, module, log } = params
   const provider = <KubernetesProvider>ctx.provider
+  const systemNamespace = provider.config.gardenSystemNamespace
 
   if (!(await containerHelpers.hasDockerfile(module))) {
     return {}
@@ -275,6 +275,7 @@ const buildHandlers: { [mode in ContainerBuildMode]: BuildHandler } = {
 // TODO: we should make a simple service around this instead of execing into containers
 export async function execInBuilder({ provider, log, args, timeout, podName, stdout, stderr }: BuilderExecParams) {
   const execCmd = ["exec", "-i", podName, "-c", dockerDaemonContainerName, "--", ...args]
+  const systemNamespace = provider.config.gardenSystemNamespace
 
   log.verbose(`Running: kubectl ${execCmd.join(" ")}`)
 
@@ -291,6 +292,7 @@ export async function execInBuilder({ provider, log, args, timeout, podName, std
 
 export async function getBuilderPodName(provider: KubernetesProvider, log: LogEntry) {
   const pod = await getRunningPodInDeployment(dockerDaemonDeploymentName, provider, log)
+  const systemNamespace = provider.config.gardenSystemNamespace
 
   if (!pod) {
     throw new PluginError(`Could not find running image builder`, {
@@ -312,8 +314,10 @@ interface RunKanikoParams {
 
 async function runKaniko({ provider, log, module, args, outputStream }: RunKanikoParams) {
   const api = await KubeApi.factory(log, provider)
+  const systemNamespace = provider.config.gardenSystemNamespace
+
   const podName = `kaniko-${module.name}-${Math.round(new Date().getTime())}`
-  const registryHostname = getRegistryHostname()
+  const registryHostname = getRegistryHostname(provider.config)
 
   const runner = new PodRunner({
     api,

garden-service/src/plugins/kubernetes/container/util.ts

@@ -9,7 +9,6 @@
 import { resolve } from "url"
 import { ContainerModule } from "../../container/config"
 import { getPortForward } from "../port-forward"
-import { systemNamespace } from "../system"
 import { CLUSTER_REGISTRY_DEPLOYMENT_NAME, CLUSTER_REGISTRY_PORT } from "../constants"
 import { containerHelpers } from "../../container/helpers"
 import { PluginError } from "../../../exceptions"
@@ -32,6 +31,8 @@ export async function queryRegistry(
 }
 
 export async function getRegistryPortForward(ctx: PluginContext, log: LogEntry) {
+  const systemNamespace = ctx.provider.config.gardenSystemNamespace
+
   return getPortForward({
     ctx,
     log,

garden-service/src/plugins/kubernetes/init.ts

@@ -15,7 +15,6 @@ import {
   getSystemServiceStatus,
   getSystemGarden,
   systemNamespaceUpToDate,
-  systemNamespace,
 } from "./system"
 import { GetEnvironmentStatusParams, EnvironmentStatus } from "../../types/plugin/provider/getEnvironmentStatus"
 import { PrepareEnvironmentParams, PrepareEnvironmentResult } from "../../types/plugin/provider/prepareEnvironment"
@@ -33,7 +32,7 @@ import {
 import { ConfigurationError } from "../../exceptions"
 
 // Note: We need to increment a version number here if we ever make breaking changes to the NFS provisioner StatefulSet
-const nfsStorageClass = "garden-system-nfs-v2"
+const nfsStorageClassVersion = 2
 
 /**
  * Performs the following actions to check environment status:
@@ -57,6 +56,7 @@ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusPar
   }
 
   const systemServiceNames = k8sCtx.provider.config._systemServices
+  const systemNamespace = ctx.provider.config.gardenSystemNamespace
 
   const detail = {
     projectReady,
@@ -248,6 +248,7 @@ export async function prepareSystem({
   const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log)
   const sysProvider = await sysGarden.resolveProvider(k8sCtx.provider.name)
   const sysCtx = <KubernetesPluginContext>sysGarden.getPluginContext(sysProvider)
+  const systemNamespace = ctx.provider.config.gardenSystemNamespace
 
   await sysGarden.clearBuilds()
 
@@ -298,12 +299,14 @@ export async function cleanupEnvironment({ ctx, log }: CleanupEnvironmentParams)
 }
 
 export function getKubernetesSystemVariables(config: KubernetesConfig) {
+  const nfsStorageClass = `${config.gardenSystemNamespace}-nfs-v${nfsStorageClassVersion}`
   const syncStorageClass = config.storage.sync.storageClass || nfsStorageClass
+  const systemNamespace = config.gardenSystemNamespace
 
   return {
     "namespace": systemNamespace,
 
-    "registry-hostname": getRegistryHostname(),
+    "registry-hostname": getRegistryHostname(config),
     "builder-mode": config.buildMode,
 
     "builder-limits-cpu": millicpuToString(config.resources.builder.limits.cpu),
@@ -338,6 +341,7 @@ export function getKubernetesSystemVariables(config: KubernetesConfig) {
   }
 }
 
-export function getRegistryHostname() {
+export function getRegistryHostname(config: KubernetesConfig) {
+  const systemNamespace = config.gardenSystemNamespace
   return `garden-docker-registry.${systemNamespace}.svc.cluster.local`
 }

garden-service/src/plugins/kubernetes/kubernetes.ts

@@ -17,7 +17,6 @@ import { containerHandlers } from "./container/handlers"
 import { kubernetesHandlers } from "./kubernetes-module/handlers"
 import { ConfigureProviderParams } from "../../types/plugin/provider/configureProvider"
 import { DebugInfo, GetDebugInfoParams } from "../../types/plugin/provider/getDebugInfo"
-import { systemNamespace, systemMetadataNamespace } from "./system"
 import { kubectl } from "./kubectl"
 import { KubernetesConfig, KubernetesPluginContext } from "./config"
 import { configSchema } from "./config"
@@ -33,6 +32,7 @@ import { helmModuleSpecSchema, helmModuleOutputsSchema } from "./helm/config"
 import { isNumber } from "util"
 import chalk from "chalk"
 import pluralize = require("pluralize")
+import { getSystemMetadataNamespaceName } from "./system"
 
 export async function configureProvider({
   projectName,
@@ -120,6 +120,10 @@ export async function debugInfo({ ctx, log, includeProject }: GetDebugInfoParams
   const k8sCtx = <KubernetesPluginContext>ctx
   const provider = k8sCtx.provider
   const entry = log.info({ section: ctx.provider.name, msg: "collecting provider configuration", status: "active" })
+
+  const systemNamespace = ctx.provider.config.gardenSystemNamespace
+  const systemMetadataNamespace = getSystemMetadataNamespaceName(provider.config)
+
   const namespacesList = [systemNamespace, systemMetadataNamespace]
   if (includeProject) {
     const appNamespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)

garden-service/src/plugins/kubernetes/local/config.ts

@@ -152,6 +152,7 @@ export async function configureProvider({ config, log, projectName }: ConfigureP
     deploymentRegistry,
     deploymentStrategy,
     forceSsl: false,
+    gardenSystemNamespace: config.gardenSystemNamespace,
     imagePullSecrets: config.imagePullSecrets,
     ingressHttpPort: 80,
     ingressHttpsPort: 443,

garden-service/src/plugins/kubernetes/system.ts

@@ -33,8 +33,11 @@ const SYSTEM_NAMESPACE_MIN_VERSION = "0.9.0"
 
 const systemProjectPath = join(STATIC_DIR, "kubernetes", "system")
 
-export const systemNamespace = "garden-system"
-export const systemMetadataNamespace = "garden-system--metadata"
+export const defaultSystemNamespace = "garden-system"
+
+export function getSystemMetadataNamespaceName(config: KubernetesConfig) {
+  return `${config.gardenSystemNamespace}--metadata`
+}
 
 /**
  * Note that we initialise system Garden with a custom Garden dir path. This is because
@@ -47,6 +50,8 @@ export async function getSystemGarden(
   variables: PrimitiveMap,
   log: LogEntry
 ): Promise<Garden> {
+  const systemNamespace = ctx.provider.config.gardenSystemNamespace
+
   const sysProvider: KubernetesConfig = {
     ...ctx.provider.config,
     environments: ["default"],
@@ -72,7 +77,7 @@ export async function getSystemGarden(
     },
     commandInfo: ctx.command,
     log: log.debug({
-      section: "garden-system",
+      section: "garden system",
       msg: "Initializing...",
       status: "active",
       indent: 1,

garden-service/src/plugins/kubernetes/test-results.ts

@@ -14,16 +14,12 @@ import { ModuleVersion } from "../../vcs/vcs"
 import { HelmModule } from "./helm/config"
 import { PluginContext } from "../../plugin-context"
 import { KubernetesPluginContext } from "./config"
-import { systemMetadataNamespace } from "./system"
 import { LogEntry } from "../../logger/log-entry"
 import { GetTestResultParams, TestResult } from "../../types/plugin/module/getTestResult"
 import hasha from "hasha"
 import { gardenAnnotationKey } from "../../util/string"
 import { upsertConfigMap } from "./util"
 import { trimRunOutput } from "./helm/common"
-import { ensureNamespace } from "./namespace"
-
-const testResultNamespace = systemMetadataNamespace
 
 export async function getTestResult({
   ctx,
@@ -34,7 +30,8 @@ export async function getTestResult({
 }: GetTestResultParams<ContainerModule | HelmModule>): Promise<TestResult | null> {
   const k8sCtx = <KubernetesPluginContext>ctx
   const api = await KubeApi.factory(log, k8sCtx.provider)
-  await ensureNamespace(api, testResultNamespace)
+  const testResultNamespace = k8sCtx.provider.config.gardenSystemNamespace
+
   const resultKey = getTestResultKey(k8sCtx, module, testName, testVersion)
 
   try {
@@ -94,7 +91,7 @@ export async function storeTestResult({
 }: StoreTestResultParams): Promise<TestResult> {
   const k8sCtx = <KubernetesPluginContext>ctx
   const api = await KubeApi.factory(log, k8sCtx.provider)
-  await ensureNamespace(api, testResultNamespace)
+  const testResultNamespace = k8sCtx.provider.config.gardenSystemNamespace
 
   const data: TestResult = trimRunOutput(result)
 

garden-service/src/plugins/kubernetes/util.ts

@@ -18,7 +18,6 @@ import { MAX_CONFIGMAP_DATA_SIZE } from "./constants"
 import { ContainerEnvVars } from "../container/config"
 import { ConfigurationError } from "../../exceptions"
 import { KubernetesProvider } from "./config"
-import { systemNamespace } from "./system"
 import { LogEntry } from "../../logger/log-entry"
 
 export const workloadTypes = ["Deployment", "DaemonSet", "ReplicaSet", "StatefulSet"]
@@ -375,6 +374,7 @@ export function convertDeprecatedManifestVersion(manifest: KubernetesResource):
 
 export async function getRunningPodInDeployment(deploymentName: string, provider: KubernetesProvider, log: LogEntry) {
   const api = await KubeApi.factory(log, provider)
+  const systemNamespace = provider.config.gardenSystemNamespace
 
   const status = await api.apps.readNamespacedDeployment(deploymentName, systemNamespace)
   const pods = await getPods(api, systemNamespace, status.spec.selector.matchLabels)

garden-service/static/kubernetes/system/ingress-controller/garden.yml

@@ -10,7 +10,7 @@ version: 0.25.1
 values:
   name: ingress-controller
   controller:
-    defaultBackendService: garden-system/default-backend
+    defaultBackendService: ${var.namespace}/default-backend
     kind: DaemonSet
     daemonset:
       useHostPort: true

garden-service/test/e2e-helpers.ts

@@ -9,7 +9,6 @@ import { TaskLogStatus } from "../src/logger/log-entry"
 import { JsonLogEntry } from "../src/logger/writers/json-terminal-writer"
 import { getExampleProjects } from "./helpers"
 import { WatchTestConditionState } from "./run-garden"
-import { systemMetadataNamespace } from "../src/plugins/kubernetes/system"
 
 export const parsedArgs = parseArgs(process.argv.slice(2))
 
@@ -36,10 +35,6 @@ export async function deleteExampleNamespaces(projectNames?: string[]) {
   await deleteNamespacesKubectl(namespacesToDelete)
 }
 
-export async function deleteSystemMetadataNamespace() {
-  await deleteExistingNamespacesKubectl([systemMetadataNamespace])
-}
-
 /*
  * The ...Kubectl suffixes on these two functions' names are tiresome, but they prevent accidental
  * imports of the wrong functions (and these two are usually not the ones that should be used).

garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts

@@ -21,6 +21,7 @@ import {
   ContainerService,
   ContainerServiceSpec,
 } from "../../../../../../src/plugins/container/config"
+import { defaultSystemNamespace } from "../../../../../../src/plugins/kubernetes/system"
 
 const kubeConfigEnvVar = process.env.KUBECONFIG
 const namespace = "my-namespace"
@@ -44,6 +45,7 @@ const basicConfig: KubernetesConfig = {
     namespace: "boo",
   },
   forceSsl: false,
+  gardenSystemNamespace: defaultSystemNamespace,
   imagePullSecrets: [],
   ingressClass: "nginx",
   ingressHttpPort: 80,