feat(k8s): allow setting custom kubeconfig path

docs/reference/module-types/kubernetes.md

@@ -208,7 +208,7 @@ POSIX-style path or filename to copy the directory or file(s).
 
 ### `dependencies`
 
-List of names of services that should be deployed before this chart.
+The names of any services that this service depends on at runtime, and the names of any tasks that should be executed before this service is deployed.
 
 | Type            | Required | Default |
 | --------------- | -------- | ------- |

docs/reference/providers/kubernetes.md

@@ -847,6 +847,16 @@ The external HTTPS port of the cluster's ingress controller.
 | -------- | -------- | ------- |
 | `number` | No       | `443`   |
 
+### `providers[].kubeconfig`
+
+[providers](#providers) > kubeconfig
+
+Path to kubeconfig file to use instead of the system default. Must be a POSIX-style path.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
 ### `providers[].namespace`
 
 [providers](#providers) > namespace
@@ -929,6 +939,7 @@ providers:
     ingressClass:
     ingressHttpPort: 80
     ingressHttpsPort: 443
+    kubeconfig:
     namespace:
     setupIngressController: false
 ```

garden-service/src/plugins/kubernetes/api.ts

@@ -29,6 +29,7 @@ import AsyncLock = require("async-lock")
 import request = require("request-promise")
 import requestErrors = require("request-promise/errors")
 import { safeLoad, safeDump } from "js-yaml"
+import { readFile } from "fs-extra"
 
 import { Omit } from "../../util/util"
 import { zip, omitBy, isObject, isPlainObject, keyBy } from "lodash"
@@ -37,6 +38,7 @@ import { KubernetesResource, KubernetesServerResource, KubernetesServerList } fr
 import { LogEntry } from "../../logger/log-entry"
 import { kubectl } from "./kubectl"
 import { urlJoin } from "../../util/string"
+import { KubernetesProvider } from "./config"
 
 interface ApiGroupMap {
   [groupVersion: string]: V1APIGroup
@@ -153,9 +155,9 @@ export class KubeApi {
     }
   }
 
-  static async factory(log: LogEntry, context: string) {
-    const config = await getContextConfig(log, context)
-    return new KubeApi(context, config)
+  static async factory(log: LogEntry, provider: KubernetesProvider) {
+    const config = await getContextConfig(log, provider)
+    return new KubeApi(provider.config.context, config)
   }
 
   async getApiInfo(): Promise<ApiInfo> {
@@ -338,7 +340,7 @@ export class KubeApi {
           return Reflect.get(target, name, receiver)
         }
 
-        return function(...args) {
+        return function (...args) {
           const defaultHeaders = target["defaultHeaders"]
 
           if (name.startsWith("patch")) {
@@ -375,12 +377,16 @@ function getGroupBasePath(groupId: string) {
   return groupId.includes("/") ? `/apis/${groupId}` : `/api/${groupId}`
 }
 
-export async function getKubeConfig(log: LogEntry) {
+export async function getKubeConfig(log: LogEntry, provider: KubernetesProvider) {
   let kubeConfigStr: string
 
   try {
-    // We use kubectl for this, to support merging multiple paths in the KUBECONFIG env var
-    kubeConfigStr = await kubectl.stdout({ log, args: ["config", "view", "--raw"] })
+    if (provider.config.kubeconfig) {
+      kubeConfigStr = (await readFile(provider.config.kubeconfig)).toString()
+    } else {
+      // We use kubectl for this, to support merging multiple paths in the KUBECONFIG env var
+      kubeConfigStr = await kubectl.stdout({ log, provider, args: ["config", "view", "--raw"] })
+    }
     return safeLoad(kubeConfigStr)
   } catch (error) {
     throw new RuntimeError(`Unable to load kubeconfig: ${error}`, {
@@ -389,12 +395,16 @@ export async function getKubeConfig(log: LogEntry) {
   }
 }
 
-async function getContextConfig(log: LogEntry, context: string): Promise<KubeConfig> {
-  if (cachedConfigs[context]) {
-    return cachedConfigs[context]
+async function getContextConfig(log: LogEntry, provider: KubernetesProvider): Promise<KubeConfig> {
+  const kubeconfigPath = provider.config.kubeconfig
+  const context = provider.config.context
+  const cacheKey = kubeconfigPath ? `${kubeconfigPath}:${context}` : context
+
+  if (cachedConfigs[cacheKey]) {
+    return cachedConfigs[cacheKey]
   }
 
-  const rawConfig = await getKubeConfig(log)
+  const rawConfig = await getKubeConfig(log, provider)
   const kc = new KubeConfig()
 
   // There doesn't appear to be a method to just load the parsed config :/

garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts

@@ -53,7 +53,7 @@ export const cleanupClusterRegistry: PluginCommand = {
     await cleanupBuildSyncVolume(provider, log)
 
     // Scan through all Pods in cluster
-    const api = await KubeApi.factory(log, provider.config.context)
+    const api = await KubeApi.factory(log, provider)
     const imagesInUse = await getImagesInUse(api, provider, log)
 
     // Get images in registry
@@ -219,7 +219,7 @@ async function runRegistryGarbageCollection(ctx: KubernetesPluginContext, api: K
   })
   delete modifiedDeployment.status
 
-  await apply({ log, context: provider.config.context, manifests: [modifiedDeployment], namespace: systemNamespace })
+  await apply({ log, provider, manifests: [modifiedDeployment], namespace: systemNamespace })
 
   // -> Wait for registry to be up again
   await waitForResources({ ctx, provider, log, serviceName: "docker-registry", resources: [modifiedDeployment] })
@@ -249,7 +249,7 @@ async function runRegistryGarbageCollection(ctx: KubernetesPluginContext, api: K
 
   await apply({
     log,
-    context: provider.config.context,
+    provider,
     manifests: [writableRegistry],
     namespace: systemNamespace,
   })
@@ -352,7 +352,7 @@ async function cleanupBuildSyncVolume(provider: KubernetesProvider, log: LogEntr
 // Returns the name for one of the build-sync pods in the cluster
 // (doesn't matter which one, they all use the same volume)
 async function getBuildSyncPodName(provider: KubernetesProvider, log: LogEntry) {
-  const api = await KubeApi.factory(log, provider.config.context)
+  const api = await KubeApi.factory(log, provider)
 
   const builderStatusRes = await api.apps.readNamespacedDeployment(buildSyncDeploymentName, systemNamespace)
   const builderPods = await getPods(api, systemNamespace, builderStatusRes.spec.selector.matchLabels)
@@ -375,7 +375,7 @@ async function execInBuildSync({ provider, log, args, timeout, podName }: Builde
 
   return kubectl.exec({
     args: execCmd,
-    context: provider.config.context,
+    provider,
     log,
     namespace: systemNamespace,
     timeout,

garden-service/src/plugins/kubernetes/config.ts

@@ -67,6 +67,7 @@ export interface KubernetesBaseConfig extends ProviderConfig {
   ingressHttpPort: number
   ingressHttpsPort: number
   ingressClass?: string
+  kubeconfig?: string
   namespace?: string
   resources: KubernetesResources
   storage: KubernetesStorage

garden-service/src/plugins/kubernetes/container/build.ts

@@ -237,15 +237,15 @@ export async function execInBuilder({ provider, log, args, timeout, podName }: B
 
   return kubectl.exec({
     args: execCmd,
-    context: provider.config.context,
+    provider,
     log,
     namespace: systemNamespace,
     timeout,
   })
 }
 
 export async function getBuilderPodName(provider: KubernetesProvider, log: LogEntry) {
-  const api = await KubeApi.factory(log, provider.config.context)
+  const api = await KubeApi.factory(log, provider)
 
   const builderStatusRes = await api.apps.readNamespacedDeployment(dockerDaemonDeploymentName, systemNamespace)
   const builderPods = await getPods(api, systemNamespace, builderStatusRes.spec.selector.matchLabels)
@@ -266,7 +266,7 @@ async function runKaniko(provider: KubernetesProvider, log: LogEntry, module: Co
   const registryHostname = getRegistryHostname()
 
   return runPod({
-    context: provider.config.context,
+    provider,
     ignoreError: false,
     image: kanikoImage,
     interactive: false,

garden-service/src/plugins/kubernetes/container/deployment.ts

@@ -42,10 +42,10 @@ export async function deployContainerService(params: DeployServiceParams<Contain
   const manifests = await createContainerObjects(k8sCtx, log, service, runtimeContext, hotReload)
 
   // TODO: use Helm instead of kubectl apply
-  const context = k8sCtx.provider.config.context
+  const provider = k8sCtx.provider
   const pruneSelector = "service=" + service.name
 
-  await apply({ log, context, manifests, force, namespace, pruneSelector })
+  await apply({ log, provider, manifests, force, namespace, pruneSelector })
 
   await waitForResources({
     ctx: k8sCtx,
@@ -69,7 +69,7 @@ export async function createContainerObjects(
   const version = service.module.version
   const provider = k8sCtx.provider
   const namespace = await getAppNamespace(k8sCtx, log, provider)
-  const api = await KubeApi.factory(log, provider.config.context)
+  const api = await KubeApi.factory(log, provider)
   const ingresses = await createIngressResources(api, provider, namespace, service)
   const deployment = await createDeployment({ provider, service, runtimeContext, namespace, enableHotReload, log })
   const kubeservices = await createServiceResources(service, namespace)
@@ -412,11 +412,10 @@ export async function deleteService(params: DeleteServiceParams): Promise<Servic
   const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)
   const provider = k8sCtx.provider
 
-  const context = provider.config.context
-  await deleteContainerDeployment({ namespace, context, serviceName: service.name, log })
+  await deleteContainerDeployment({ namespace, provider, serviceName: service.name, log })
   await deleteObjectsByLabel({
     log,
-    context,
+    provider,
     namespace,
     labelKey: "service",
     labelValue: service.name,
@@ -428,11 +427,12 @@ export async function deleteService(params: DeleteServiceParams): Promise<Servic
 }
 
 export async function deleteContainerDeployment(
-  { namespace, context, serviceName, log }: { namespace: string, context: string, serviceName: string, log: LogEntry },
+  { namespace, provider, serviceName, log }:
+    { namespace: string, provider: KubernetesProvider, serviceName: string, log: LogEntry },
 ) {
 
   let found = true
-  const api = await KubeApi.factory(log, context)
+  const api = await KubeApi.factory(log, provider)
 
   try {
     await api.extensions.deleteNamespacedDeployment(serviceName, namespace, <any>{})

garden-service/src/plugins/kubernetes/container/logs.ts

@@ -17,11 +17,11 @@ import { emptyRuntimeContext } from "../../../runtime-context"
 export async function getServiceLogs(params: GetServiceLogsParams<ContainerModule>) {
   const { ctx, log, service } = params
   const k8sCtx = <KubernetesPluginContext>ctx
-  const context = k8sCtx.provider.config.context
-  const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)
+  const provider = k8sCtx.provider
+  const namespace = await getAppNamespace(k8sCtx, log, provider)
 
   const resources = [await createDeployment({
-    provider: k8sCtx.provider,
+    provider,
     service,
     // No need for the proper context here
     runtimeContext: emptyRuntimeContext,
@@ -30,5 +30,5 @@ export async function getServiceLogs(params: GetServiceLogsParams<ContainerModul
     log,
   })]
 
-  return getAllLogs({ ...params, context, namespace, resources })
+  return getAllLogs({ ...params, provider, namespace, resources })
 }

garden-service/src/plugins/kubernetes/container/run.ts

@@ -64,7 +64,7 @@ export async function execInDeployment(
       interactive: boolean,
     },
 ) {
-  const api = await KubeApi.factory(log, provider.config.context)
+  const api = await KubeApi.factory(log, provider)
   const deployment = await api.apps.readNamespacedDeployment(deploymentName, namespace)
   const pods = await getWorkloadPods(api, namespace, deployment)
 
@@ -87,7 +87,7 @@ export async function execInDeployment(
   const kubecmd = ["exec", ...opts, pod.metadata.name, "--", ...command]
   const res = await kubectl.spawnAndWait({
     log,
-    context: api.context,
+    provider,
     namespace,
     args: kubecmd,
     ignoreError: true,
@@ -104,7 +104,6 @@ export async function runContainerModule(
   }: RunModuleParams<ContainerModule>,
 ): Promise<RunResult> {
   const provider = <KubernetesProvider>ctx.provider
-  const context = provider.config.context
   const namespace = await getAppNamespace(ctx, log, provider)
 
   // Apply overrides
@@ -123,7 +122,7 @@ export async function runContainerModule(
   }
 
   return runPod({
-    context,
+    provider,
     image,
     interactive,
     ignoreError,
@@ -155,7 +154,6 @@ export async function runContainerTask(
   { ctx, log, module, task, taskVersion, interactive, runtimeContext }: RunTaskParams<ContainerModule>,
 ): Promise<RunTaskResult> {
   const provider = <KubernetesProvider>ctx.provider
-  const context = provider.config.context
   const namespace = await getAppNamespace(ctx, log, provider)
 
   // Apply overrides
@@ -175,7 +173,7 @@ export async function runContainerTask(
   }
 
   const res = await runPod({
-    context,
+    provider,
     image,
     interactive,
     ignoreError: false,

garden-service/src/plugins/kubernetes/container/status.ts

@@ -30,7 +30,7 @@ export async function getContainerServiceStatus(
   // TODO: hash and compare all the configuration files (otherwise internal changes don't get deployed)
   const version = module.version
   const provider = k8sCtx.provider
-  const api = await KubeApi.factory(log, provider.config.context)
+  const api = await KubeApi.factory(log, provider)
   const namespace = await getAppNamespace(k8sCtx, log, provider)
 
   // FIXME: [objects, matched] and ingresses can be run in parallel

garden-service/src/plugins/kubernetes/container/test.ts

@@ -45,7 +45,7 @@ export async function testContainerModule(
   }
 
   const result = await runPod({
-    context: provider.config.context,
+    provider,
     image,
     interactive,
     ignoreError: true, // to ensure results get stored when an error occurs

garden-service/src/plugins/kubernetes/helm/build.ts

@@ -24,19 +24,18 @@ export async function buildHelmModule({ ctx, module, log }: BuildModuleParams<He
     provider: k8sCtx.provider,
     skipCreate: true,
   })
-  const context = ctx.provider.config.context
   const baseModule = getBaseModule(module)
 
   if (!baseModule && !(await containsSource(module))) {
     log.debug("Fetching chart...")
     try {
-      await fetchChart(namespace, context, log, module)
+      await fetchChart(k8sCtx, namespace, log, module)
     } catch {
       // update the local helm repo and retry
       log.debug("Updating Helm repo...")
-      await helm(namespace, context, log, ...["repo", "update"])
+      await helm({ ctx: k8sCtx, namespace, log, args: [...["repo", "update"]] })
       log.debug("Fetching chart (after updating)...")
-      await fetchChart(namespace, context, log, module)
+      await fetchChart(k8sCtx, namespace, log, module)
     }
   }
 
@@ -54,10 +53,10 @@ export async function buildHelmModule({ ctx, module, log }: BuildModuleParams<He
   return { fresh: true }
 }
 
-async function fetchChart(namespace: string, context: string, log: LogEntry, module: HelmModule) {
+async function fetchChart(ctx: KubernetesPluginContext, namespace: string, log: LogEntry, module: HelmModule) {
   const buildPath = module.buildPath
 
-  await helm(namespace, context, log, "init", "--client-only")
+  await helm({ ctx, namespace, log, args: ["init", "--client-only"] })
 
   const fetchArgs = [
     "fetch", module.spec.chart!,
@@ -70,5 +69,5 @@ async function fetchChart(namespace: string, context: string, log: LogEntry, mod
   if (module.spec.repo) {
     fetchArgs.push("--repo", module.spec.repo)
   }
-  await helm(namespace, context, log, ...fetchArgs)
+  await helm({ ctx, namespace, log, args: [...fetchArgs] })
 }