From acf8a5ebd714ce58ebcee7143c6d94f7462162e0 Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Fri, 29 Nov 2019 23:48:46 +0100 Subject: [PATCH] refactor(k8s): allow overriding the default garden-system namespace Flagging as a refactor because this is mainly intended for internal usage for now, since changing the namespace might have some unintended consequences. --- garden-service/src/garden.ts | 1 + .../kubernetes/commands/cleanup-cluster-registry.ts | 4 +++- .../src/plugins/kubernetes/commands/cluster-init.ts | 5 +++-- garden-service/src/plugins/kubernetes/config.ts | 12 ++++++++++++ .../src/plugins/kubernetes/container/build.ts | 8 ++++++-- .../src/plugins/kubernetes/container/util.ts | 3 ++- garden-service/src/plugins/kubernetes/init.ts | 12 ++++++++---- garden-service/src/plugins/kubernetes/kubernetes.ts | 6 +++++- .../src/plugins/kubernetes/local/config.ts | 1 + garden-service/src/plugins/kubernetes/system.ts | 11 ++++++++--- .../src/plugins/kubernetes/test-results.ts | 9 +++------ garden-service/src/plugins/kubernetes/util.ts | 2 +- .../kubernetes/system/ingress-controller/garden.yml | 2 +- garden-service/test/e2e-helpers.ts | 5 ----- .../unit/src/plugins/kubernetes/container/ingress.ts | 2 ++ 15 files changed, 56 insertions(+), 27 deletions(-) diff --git a/garden-service/src/garden.ts b/garden-service/src/garden.ts index a736060e2a..205dcc74e4 100644 --- a/garden-service/src/garden.ts +++ b/garden-service/src/garden.ts @@ -143,6 +143,7 @@ export class Garden { public readonly moduleIncludePatterns?: string[] public readonly moduleExcludePatterns: string[] public readonly persistent: boolean + public readonly systemNamespace: string constructor(params: GardenParams) { this.buildDir = params.buildDir diff --git a/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts b/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts index f79523ff1b..0cbe4368ff 100644 --- a/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts +++ b/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts @@ -20,7 +20,6 @@ import { splitFirst, splitLast } from "../../../util/util" import { LogEntry } from "../../../logger/log-entry" import Bluebird from "bluebird" import { CLUSTER_REGISTRY_DEPLOYMENT_NAME } from "../constants" -import { systemNamespace } from "../system" import { PluginError } from "../../../exceptions" import { apply, kubectl } from "../kubectl" import { waitForResources } from "../status/status" @@ -199,6 +198,7 @@ async function runRegistryGarbageCollection(ctx: KubernetesPluginContext, api: K }) const provider = ctx.provider + const systemNamespace = provider.config.gardenSystemNamespace // Restart the registry in read-only mode // -> Get the original deployment @@ -404,6 +404,7 @@ async function cleanupBuildSyncVolume(provider: KubernetesProvider, log: LogEntr // (doesn't matter which one, they all use the same volume) async function getBuildSyncPodName(provider: KubernetesProvider, log: LogEntry) { const api = await KubeApi.factory(log, provider) + const systemNamespace = provider.config.gardenSystemNamespace const builderStatusRes = await api.apps.readNamespacedDeployment(buildSyncDeploymentName, systemNamespace) const builderPods = await getPods(api, systemNamespace, builderStatusRes.spec.selector.matchLabels) @@ -421,6 +422,7 @@ async function getBuildSyncPodName(provider: KubernetesProvider, log: LogEntry) async function execInBuildSync({ provider, log, args, timeout, podName }: BuilderExecParams) { const execCmd = ["exec", "-i", podName, "--", ...args] + const systemNamespace = provider.config.gardenSystemNamespace log.verbose(`Running: kubectl ${execCmd.join(" ")}`) diff --git a/garden-service/src/plugins/kubernetes/commands/cluster-init.ts b/garden-service/src/plugins/kubernetes/commands/cluster-init.ts index 7ce556db28..2d18a2aa61 100644 --- a/garden-service/src/plugins/kubernetes/commands/cluster-init.ts +++ b/garden-service/src/plugins/kubernetes/commands/cluster-init.ts @@ -10,7 +10,7 @@ import { PluginCommand } from "../../../types/plugin/command" import { prepareSystem, getEnvironmentStatus } from "../init" import chalk from "chalk" import { helm } from "../helm/helm-cli" -import { KubernetesPluginContext } from "../config" +import { KubernetesPluginContext, KubernetesProvider } from "../config" export const clusterInit: PluginCommand = { name: "cluster-init", @@ -21,6 +21,7 @@ export const clusterInit: PluginCommand = { }, handler: async ({ ctx, log }) => { + const provider = ctx.provider as KubernetesProvider const status = await getEnvironmentStatus({ ctx, log }) let result = {} @@ -44,7 +45,7 @@ export const clusterInit: PluginCommand = { await helm({ ctx: k8sCtx, log, - namespace: "garden-system", + namespace: provider.config.gardenSystemNamespace, args: ["delete", "--purge", "garden-nfs-provisioner"], }) } catch (_) {} diff --git a/garden-service/src/plugins/kubernetes/config.ts b/garden-service/src/plugins/kubernetes/config.ts index e97f84a1ae..f7614eb6fd 100644 --- a/garden-service/src/plugins/kubernetes/config.ts +++ b/garden-service/src/plugins/kubernetes/config.ts @@ -13,6 +13,7 @@ import { Provider, providerConfigBaseSchema, ProviderConfig } from "../../config import { containerRegistryConfigSchema, ContainerRegistryConfig } from "../container/config" import { PluginContext } from "../../plugin-context" import { deline } from "../../util/string" +import { defaultSystemNamespace } from "./system" export interface ProviderSecretRef { name: string @@ -100,6 +101,7 @@ export interface KubernetesBaseConfig extends ProviderConfig { registryProxyTolerations: Toleration[] resources: KubernetesResources storage: KubernetesStorage + gardenSystemNamespace: string tlsCertificates: IngressTlsCertificate[] certManager?: CertManagerConfig _systemServices: string[] @@ -345,6 +347,16 @@ export const kubernetesConfigBase = providerConfigBaseSchema.keys({ "Require SSL on all `container` module services. If set to true, an error is raised when no certificate " + "is available for a configured hostname on a `container` module." ), + gardenSystemNamespace: joi + .string() + .default(defaultSystemNamespace) + .description( + dedent` + Override the garden-system namespace name. This option is mainly used for testing. + In most cases you should leave the default value. + ` + ) + .meta({ internal: true }), imagePullSecrets: imagePullSecretsSchema, // TODO: invert the resources and storage config schemas resources: joi diff --git a/garden-service/src/plugins/kubernetes/container/build.ts b/garden-service/src/plugins/kubernetes/container/build.ts index 71178ba46e..8ef77c142c 100644 --- a/garden-service/src/plugins/kubernetes/container/build.ts +++ b/garden-service/src/plugins/kubernetes/container/build.ts @@ -14,7 +14,6 @@ import { buildContainerModule, getContainerBuildStatus, getDockerBuildFlags } fr import { GetBuildStatusParams, BuildStatus } from "../../../types/plugin/module/getBuildStatus" import { BuildModuleParams, BuildResult } from "../../../types/plugin/module/build" import { millicpuToString, megabytesToString, getRunningPodInDeployment } from "../util" -import { systemNamespace } from "../system" import { RSYNC_PORT } from "../constants" import { posix, resolve } from "path" import { KubeApi } from "../api" @@ -118,6 +117,7 @@ const localBuild: BuildHandler = async (params) => { const remoteBuild: BuildHandler = async (params) => { const { ctx, module, log } = params const provider = ctx.provider + const systemNamespace = provider.config.gardenSystemNamespace if (!(await containerHelpers.hasDockerfile(module))) { return {} @@ -274,6 +274,7 @@ const buildHandlers: { [mode in ContainerBuildMode]: BuildHandler } = { // TODO: we should make a simple service around this instead of execing into containers export async function execInBuilder({ provider, log, args, timeout, podName, stdout, stderr }: BuilderExecParams) { const execCmd = ["exec", "-i", podName, "-c", dockerDaemonContainerName, "--", ...args] + const systemNamespace = provider.config.gardenSystemNamespace log.verbose(`Running: kubectl ${execCmd.join(" ")}`) @@ -290,6 +291,7 @@ export async function execInBuilder({ provider, log, args, timeout, podName, std export async function getBuilderPodName(provider: KubernetesProvider, log: LogEntry) { const pod = await getRunningPodInDeployment(dockerDaemonDeploymentName, provider, log) + const systemNamespace = provider.config.gardenSystemNamespace if (!pod) { throw new PluginError(`Could not find running image builder`, { @@ -311,8 +313,10 @@ interface RunKanikoParams { async function runKaniko({ provider, log, module, args, outputStream }: RunKanikoParams) { const api = await KubeApi.factory(log, provider) + const systemNamespace = provider.config.gardenSystemNamespace + const podName = `kaniko-${module.name}-${Math.round(new Date().getTime())}` - const registryHostname = getRegistryHostname() + const registryHostname = getRegistryHostname(provider.config) const k8sSystemVars = getKubernetesSystemVariables(provider.config) const syncDataVolumeName = k8sSystemVars["sync-volume-name"] diff --git a/garden-service/src/plugins/kubernetes/container/util.ts b/garden-service/src/plugins/kubernetes/container/util.ts index 44632c527c..cc574fa204 100644 --- a/garden-service/src/plugins/kubernetes/container/util.ts +++ b/garden-service/src/plugins/kubernetes/container/util.ts @@ -9,7 +9,6 @@ import { resolve } from "url" import { ContainerModule } from "../../container/config" import { getPortForward } from "../port-forward" -import { systemNamespace } from "../system" import { CLUSTER_REGISTRY_DEPLOYMENT_NAME, CLUSTER_REGISTRY_PORT } from "../constants" import { containerHelpers } from "../../container/helpers" import { PluginError } from "../../../exceptions" @@ -32,6 +31,8 @@ export async function queryRegistry( } export async function getRegistryPortForward(ctx: PluginContext, log: LogEntry) { + const systemNamespace = ctx.provider.config.gardenSystemNamespace + return getPortForward({ ctx, log, diff --git a/garden-service/src/plugins/kubernetes/init.ts b/garden-service/src/plugins/kubernetes/init.ts index 2f5bbd91a1..713e6b6bf4 100644 --- a/garden-service/src/plugins/kubernetes/init.ts +++ b/garden-service/src/plugins/kubernetes/init.ts @@ -15,7 +15,6 @@ import { getSystemServiceStatus, getSystemGarden, systemNamespaceUpToDate, - systemNamespace, } from "./system" import { GetEnvironmentStatusParams, EnvironmentStatus } from "../../types/plugin/provider/getEnvironmentStatus" import { PrepareEnvironmentParams, PrepareEnvironmentResult } from "../../types/plugin/provider/prepareEnvironment" @@ -33,7 +32,7 @@ import { import { ConfigurationError } from "../../exceptions" // Note: We need to increment a version number here if we ever make breaking changes to the NFS provisioner StatefulSet -const nfsStorageClass = "garden-system-nfs-v2" +const nfsStorageClassVersion = 2 /** * Performs the following actions to check environment status: @@ -57,6 +56,7 @@ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusPar } const systemServiceNames = k8sCtx.provider.config._systemServices + const systemNamespace = ctx.provider.config.gardenSystemNamespace const detail = { projectReady, @@ -248,6 +248,7 @@ export async function prepareSystem({ const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log) const sysProvider = await sysGarden.resolveProvider(k8sCtx.provider.name) const sysCtx = sysGarden.getPluginContext(sysProvider) + const systemNamespace = ctx.provider.config.gardenSystemNamespace await sysGarden.clearBuilds() @@ -298,12 +299,14 @@ export async function cleanupEnvironment({ ctx, log }: CleanupEnvironmentParams) } export function getKubernetesSystemVariables(config: KubernetesConfig) { + const nfsStorageClass = `${config.gardenSystemNamespace}-nfs-v${nfsStorageClassVersion}` const syncStorageClass = config.storage.sync.storageClass || nfsStorageClass + const systemNamespace = config.gardenSystemNamespace return { "namespace": systemNamespace, - "registry-hostname": getRegistryHostname(), + "registry-hostname": getRegistryHostname(config), "builder-mode": config.buildMode, "builder-limits-cpu": millicpuToString(config.resources.builder.limits.cpu), @@ -338,6 +341,7 @@ export function getKubernetesSystemVariables(config: KubernetesConfig) { } } -export function getRegistryHostname() { +export function getRegistryHostname(config: KubernetesConfig) { + const systemNamespace = config.gardenSystemNamespace return `garden-docker-registry.${systemNamespace}.svc.cluster.local` } diff --git a/garden-service/src/plugins/kubernetes/kubernetes.ts b/garden-service/src/plugins/kubernetes/kubernetes.ts index 75e6581a9c..4016052a2a 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes.ts @@ -17,7 +17,6 @@ import { containerHandlers } from "./container/handlers" import { kubernetesHandlers } from "./kubernetes-module/handlers" import { ConfigureProviderParams } from "../../types/plugin/provider/configureProvider" import { DebugInfo, GetDebugInfoParams } from "../../types/plugin/provider/getDebugInfo" -import { systemNamespace, systemMetadataNamespace } from "./system" import { kubectl } from "./kubectl" import { KubernetesConfig, KubernetesPluginContext } from "./config" import { configSchema } from "./config" @@ -33,6 +32,7 @@ import { helmModuleSpecSchema, helmModuleOutputsSchema } from "./helm/config" import { isNumber } from "util" import chalk from "chalk" import pluralize = require("pluralize") +import { getSystemMetadataNamespaceName } from "./system" export async function configureProvider({ projectName, @@ -120,6 +120,10 @@ export async function debugInfo({ ctx, log, includeProject }: GetDebugInfoParams const k8sCtx = ctx const provider = k8sCtx.provider const entry = log.info({ section: ctx.provider.name, msg: "collecting provider configuration", status: "active" }) + + const systemNamespace = ctx.provider.config.gardenSystemNamespace + const systemMetadataNamespace = getSystemMetadataNamespaceName(provider.config) + const namespacesList = [systemNamespace, systemMetadataNamespace] if (includeProject) { const appNamespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider) diff --git a/garden-service/src/plugins/kubernetes/local/config.ts b/garden-service/src/plugins/kubernetes/local/config.ts index 787c0417b4..43f7678316 100644 --- a/garden-service/src/plugins/kubernetes/local/config.ts +++ b/garden-service/src/plugins/kubernetes/local/config.ts @@ -152,6 +152,7 @@ export async function configureProvider({ config, log, projectName }: ConfigureP deploymentRegistry, deploymentStrategy, forceSsl: false, + gardenSystemNamespace: config.gardenSystemNamespace, imagePullSecrets: config.imagePullSecrets, ingressHttpPort: 80, ingressHttpsPort: 443, diff --git a/garden-service/src/plugins/kubernetes/system.ts b/garden-service/src/plugins/kubernetes/system.ts index df79bdbee3..82469bd2ca 100644 --- a/garden-service/src/plugins/kubernetes/system.ts +++ b/garden-service/src/plugins/kubernetes/system.ts @@ -34,8 +34,11 @@ const SYSTEM_NAMESPACE_MIN_VERSION = "0.9.0" const systemProjectPath = join(STATIC_DIR, "kubernetes", "system") -export const systemNamespace = "garden-system" -export const systemMetadataNamespace = "garden-system--metadata" +export const defaultSystemNamespace = "garden-system" + +export function getSystemMetadataNamespaceName(config: KubernetesConfig) { + return `${config.gardenSystemNamespace}--metadata` +} /** * Note that we initialise system Garden with a custom Garden dir path. This is because @@ -48,6 +51,8 @@ export async function getSystemGarden( variables: PrimitiveMap, log: LogEntry ): Promise { + const systemNamespace = ctx.provider.config.gardenSystemNamespace + const sysProvider: KubernetesConfig = { ...ctx.provider.config, environments: ["default"], @@ -73,7 +78,7 @@ export async function getSystemGarden( }, commandInfo: ctx.command, log: log.debug({ - section: "garden-system", + section: "garden system", msg: "Initializing...", status: "active", indent: 1, diff --git a/garden-service/src/plugins/kubernetes/test-results.ts b/garden-service/src/plugins/kubernetes/test-results.ts index 8b2763a6b5..926b09c9f4 100644 --- a/garden-service/src/plugins/kubernetes/test-results.ts +++ b/garden-service/src/plugins/kubernetes/test-results.ts @@ -14,16 +14,12 @@ import { ModuleVersion } from "../../vcs/vcs" import { HelmModule } from "./helm/config" import { PluginContext } from "../../plugin-context" import { KubernetesPluginContext } from "./config" -import { systemMetadataNamespace } from "./system" import { LogEntry } from "../../logger/log-entry" import { GetTestResultParams, TestResult } from "../../types/plugin/module/getTestResult" import hasha from "hasha" import { gardenAnnotationKey } from "../../util/string" import { upsertConfigMap } from "./util" import { trimRunOutput } from "./helm/common" -import { ensureNamespace } from "./namespace" - -const testResultNamespace = systemMetadataNamespace export async function getTestResult({ ctx, @@ -34,7 +30,8 @@ export async function getTestResult({ }: GetTestResultParams): Promise { const k8sCtx = ctx const api = await KubeApi.factory(log, k8sCtx.provider) - await ensureNamespace(api, testResultNamespace) + const testResultNamespace = k8sCtx.provider.config.gardenSystemNamespace + const resultKey = getTestResultKey(k8sCtx, module, testName, testVersion) try { @@ -94,7 +91,7 @@ export async function storeTestResult({ }: StoreTestResultParams): Promise { const k8sCtx = ctx const api = await KubeApi.factory(log, k8sCtx.provider) - await ensureNamespace(api, testResultNamespace) + const testResultNamespace = k8sCtx.provider.config.gardenSystemNamespace const data: TestResult = trimRunOutput(result) diff --git a/garden-service/src/plugins/kubernetes/util.ts b/garden-service/src/plugins/kubernetes/util.ts index be847b2a97..7d9bbe3a60 100644 --- a/garden-service/src/plugins/kubernetes/util.ts +++ b/garden-service/src/plugins/kubernetes/util.ts @@ -18,7 +18,6 @@ import { MAX_CONFIGMAP_DATA_SIZE } from "./constants" import { ContainerEnvVars } from "../container/config" import { ConfigurationError } from "../../exceptions" import { KubernetesProvider } from "./config" -import { systemNamespace } from "./system" import { LogEntry } from "../../logger/log-entry" export const workloadTypes = ["Deployment", "DaemonSet", "ReplicaSet", "StatefulSet"] @@ -375,6 +374,7 @@ export function convertDeprecatedManifestVersion(manifest: KubernetesResource): export async function getRunningPodInDeployment(deploymentName: string, provider: KubernetesProvider, log: LogEntry) { const api = await KubeApi.factory(log, provider) + const systemNamespace = provider.config.gardenSystemNamespace const status = await api.apps.readNamespacedDeployment(deploymentName, systemNamespace) const pods = await getPods(api, systemNamespace, status.spec.selector.matchLabels) diff --git a/garden-service/static/kubernetes/system/ingress-controller/garden.yml b/garden-service/static/kubernetes/system/ingress-controller/garden.yml index 4d86a5a5c3..8ede4c294b 100644 --- a/garden-service/static/kubernetes/system/ingress-controller/garden.yml +++ b/garden-service/static/kubernetes/system/ingress-controller/garden.yml @@ -10,7 +10,7 @@ version: 0.25.1 values: name: ingress-controller controller: - defaultBackendService: garden-system/default-backend + defaultBackendService: ${var.namespace}/default-backend kind: DaemonSet daemonset: useHostPort: true diff --git a/garden-service/test/e2e-helpers.ts b/garden-service/test/e2e-helpers.ts index 09797b2953..22506deb1e 100644 --- a/garden-service/test/e2e-helpers.ts +++ b/garden-service/test/e2e-helpers.ts @@ -9,7 +9,6 @@ import { TaskLogStatus } from "../src/logger/log-entry" import { JsonLogEntry } from "../src/logger/writers/json-terminal-writer" import { getExampleProjects } from "./helpers" import { WatchTestConditionState } from "./run-garden" -import { systemMetadataNamespace } from "../src/plugins/kubernetes/system" export const parsedArgs = parseArgs(process.argv.slice(2)) @@ -36,10 +35,6 @@ export async function deleteExampleNamespaces(projectNames?: string[]) { await deleteNamespacesKubectl(namespacesToDelete) } -export async function deleteSystemMetadataNamespace() { - await deleteExistingNamespacesKubectl([systemMetadataNamespace]) -} - /* * The ...Kubectl suffixes on these two functions' names are tiresome, but they prevent accidental * imports of the wrong functions (and these two are usually not the ones that should be used). diff --git a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts index a660b1aed5..749156be1d 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts @@ -21,6 +21,7 @@ import { ContainerService, ContainerServiceSpec, } from "../../../../../../src/plugins/container/config" +import { defaultSystemNamespace } from "../../../../../../src/plugins/kubernetes/system" const kubeConfigEnvVar = process.env.KUBECONFIG const namespace = "my-namespace" @@ -44,6 +45,7 @@ const basicConfig: KubernetesConfig = { namespace: "boo", }, forceSsl: false, + gardenSystemNamespace: defaultSystemNamespace, imagePullSecrets: [], ingressClass: "nginx", ingressHttpPort: 80,