fix(k8s): allow multiple paths in KUBECONFIG env var

We now use kubectl to merge them, to keep things nice and consistent.

Closes #711

garden-service/src/plugins/kubernetes/api.ts

@@ -16,22 +16,16 @@ import {
   V1Secret,
   Policy_v1beta1Api,
 } from "@kubernetes/client-node"
-import { join } from "path"
 import request = require("request-promise")
-import { readFileSync, pathExistsSync } from "fs-extra"
-import { safeLoad } from "js-yaml"
+import { safeLoad, safeDump } from "js-yaml"
 import { zip, omitBy, isObject } from "lodash"
-import { GardenBaseError } from "../../exceptions"
-import { homedir } from "os"
+import { GardenBaseError, RuntimeError, ConfigurationError } from "../../exceptions"
 import { KubernetesResource } from "./types"
-import * as dedent from "dedent"
 import { LogEntry } from "../../logger/log-entry"
 import { splitLast, findByName } from "../../util/util"
+import { kubectl } from "./kubectl"
 
-let kubeConfigStr: string
-let kubeConfig: any
-
-const configs: { [context: string]: KubeConfig } = {}
+const cachedConfigs: { [context: string]: KubeConfig } = {}
 
 // NOTE: be warned, the API of the client library is very likely to change
 
@@ -73,24 +67,34 @@ export class KubernetesError extends GardenBaseError {
 }
 
 export class KubeApi {
-  private config: KubeConfig
-
   public apiExtensions: Apiextensions_v1beta1Api
   public apps: Apps_v1Api
   public core: Core_v1Api
   public extensions: Extensions_v1beta1Api
   public policy: Policy_v1beta1Api
   public rbac: RbacAuthorization_v1Api
 
-  constructor(public context: string) {
-    this.config = getConfig(this.context)
+  constructor(public context: string, private config: KubeConfig) {
+    const cluster = this.config.getCurrentCluster()
+
+    if (!cluster) {
+      throw new ConfigurationError(`Could not read cluster from kubeconfig for context ${context}`, {
+        context,
+        config,
+      })
+    }
 
     for (const [name, cls] of Object.entries(apiTypes)) {
-      const api = new cls(this.config.getCurrentCluster()!.server)
+      const api = new cls(cluster.server)
       this[name] = this.proxyApi(api, this.config)
     }
   }
 
+  static async factory(log: LogEntry, context: string) {
+    const config = await getContextConfig(log, context)
+    return new KubeApi(context, config)
+  }
+
   async readBySpec(namespace: string, spec: KubernetesResource, log: LogEntry) {
     // this is just awful, sorry. any better ideas? - JE
     const name = spec.metadata.name
@@ -245,7 +249,7 @@ export class KubeApi {
   /**
    * Wrapping the API objects to deal with bugs.
    */
-  private proxyApi<T extends K8sApi>(api: T, config): T {
+  private proxyApi<T extends K8sApi>(api: T, config: KubeConfig): T {
     api.setDefaultAuthentication(config)
 
     return new Proxy(api, {
@@ -254,7 +258,7 @@ export class KubeApi {
           return Reflect.get(target, name, receiver)
         }
 
-        return function(...args) {
+        return function (...args) {
           const defaultHeaders = target["defaultHeaders"]
 
           if (name.startsWith("patch")) {
@@ -277,41 +281,42 @@ export class KubeApi {
   }
 }
 
-function getConfig(context: string): KubeConfig {
-  const kubeConfigPath = process.env.KUBECONFIG || join(homedir(), ".kube", "config")
+export async function getKubeConfig(log: LogEntry) {
+  let kubeConfigStr: string
 
-  if (pathExistsSync(kubeConfigPath)) {
-    kubeConfigStr = readFileSync(kubeConfigPath).toString()
-  } else {
-    // Fall back to a blank kubeconfig if none is found
-    kubeConfigStr = dedent`
-      apiVersion: v1
-      kind: Config
-      clusters: []
-      contexts: []
-      preferences: {}
-      users: []
-    `
+  try {
+    // We use kubectl for this, to support merging multiple paths in the KUBECONFIG env var
+    kubeConfigStr = await kubectl.stdout({ log, args: ["config", "view", "--raw"] })
+    return safeLoad(kubeConfigStr)
+  } catch (error) {
+    throw new RuntimeError(`Unable to load kubeconfig: ${error}`, {
+      error,
+    })
   }
-  kubeConfig = safeLoad(kubeConfigStr)
+}
 
-  if (!configs[context]) {
-    const kc = new KubeConfig()
+async function getContextConfig(log: LogEntry, context: string): Promise<KubeConfig> {
+  if (cachedConfigs[context]) {
+    return cachedConfigs[context]
+  }
 
-    kc.loadFromString(kubeConfigStr)
-    kc.setCurrentContext(context)
+  const rawConfig = await getKubeConfig(log)
+  const kc = new KubeConfig()
 
-    // FIXME: need to patch a bug in the library here (https://github.com/kubernetes-client/javascript/pull/54)
-    for (const [a, b] of zip(kubeConfig["clusters"] || [], kc.clusters)) {
-      if (a && a["cluster"]["insecure-skip-tls-verify"] === true) {
-        (<any>b).skipTLSVerify = true
-      }
-    }
+  // There doesn't appear to be a method to just load the parsed config :/
+  kc.loadFromString(safeDump(rawConfig))
+  kc.setCurrentContext(context)
 
-    configs[context] = kc
+  // FIXME: need to patch a bug in the library here (https://github.com/kubernetes-client/javascript/pull/54)
+  for (const [a, b] of zip(rawConfig["clusters"] || [], kc.clusters)) {
+    if (a && a["cluster"]["insecure-skip-tls-verify"] === true) {
+      (<any>b).skipTLSVerify = true
+    }
   }
 
-  return configs[context]
+  cachedConfigs[context] = kc
+
+  return kc
 }
 
 function wrapError(err) {

garden-service/src/plugins/kubernetes/container/deployment.ts

@@ -35,8 +35,8 @@ export async function deployContainerService(params: DeployServiceParams<Contain
   const { ctx, service, runtimeContext, force, log, hotReload } = params
   const k8sCtx = <KubernetesPluginContext>ctx
 
-  const namespace = await getAppNamespace(k8sCtx, k8sCtx.provider)
-  const manifests = await createContainerObjects(k8sCtx, service, runtimeContext, hotReload)
+  const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)
+  const manifests = await createContainerObjects(k8sCtx, log, service, runtimeContext, hotReload)
 
   // TODO: use Helm instead of kubectl apply
   const context = k8sCtx.provider.config.context
@@ -57,15 +57,16 @@ export async function deployContainerService(params: DeployServiceParams<Contain
 
 export async function createContainerObjects(
   ctx: PluginContext,
+  log: LogEntry,
   service: ContainerService,
   runtimeContext: RuntimeContext,
   enableHotReload: boolean,
 ) {
   const k8sCtx = <KubernetesPluginContext>ctx
   const version = service.module.version
   const provider = k8sCtx.provider
-  const namespace = await getAppNamespace(k8sCtx, provider)
-  const api = new KubeApi(provider.config.context)
+  const namespace = await getAppNamespace(k8sCtx, log, provider)
+  const api = await KubeApi.factory(log, provider.config.context)
   const ingresses = await createIngressResources(api, provider, namespace, service)
   const deployment = await createDeployment(provider, service, runtimeContext, namespace, enableHotReload)
   const kubeservices = await createServiceResources(service, namespace)
@@ -381,7 +382,7 @@ export function rsyncTargetPath(path: string) {
 export async function deleteService(params: DeleteServiceParams): Promise<ServiceStatus> {
   const { ctx, log, service } = params
   const k8sCtx = <KubernetesPluginContext>ctx
-  const namespace = await getAppNamespace(k8sCtx, k8sCtx.provider)
+  const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)
   const provider = k8sCtx.provider
 
   const context = provider.config.context
@@ -404,7 +405,7 @@ export async function deleteContainerDeployment(
 ) {
 
   let found = true
-  const api = new KubeApi(context)
+  const api = await KubeApi.factory(log, context)
 
   try {
     await api.extensions.deleteNamespacedDeployment(serviceName, namespace, <any>{})

garden-service/src/plugins/kubernetes/container/logs.ts

@@ -14,10 +14,10 @@ import { KubernetesPluginContext } from "../kubernetes"
 import { createDeployment } from "./deployment"
 
 export async function getServiceLogs(params: GetServiceLogsParams<ContainerModule>) {
-  const { ctx, service } = params
+  const { ctx, log, service } = params
   const k8sCtx = <KubernetesPluginContext>ctx
   const context = k8sCtx.provider.config.context
-  const namespace = await getAppNamespace(k8sCtx, k8sCtx.provider)
+  const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)
 
   const resources = [await createDeployment(k8sCtx.provider, service, params.runtimeContext, namespace, false)]
 

garden-service/src/plugins/kubernetes/container/run.ts

@@ -29,9 +29,9 @@ export async function execInService(params: ExecInServiceParams<ContainerModule>
   const { ctx, log, service, command, interactive } = params
   const k8sCtx = <KubernetesPluginContext>ctx
   const provider = k8sCtx.provider
-  const api = new KubeApi(provider.config.context)
+  const api = await KubeApi.factory(log, provider.config.context)
   const status = await getContainerServiceStatus({ ...params, hotReload: false })
-  const namespace = await getAppNamespace(k8sCtx, k8sCtx.provider)
+  const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)
 
   // TODO: this check should probably live outside of the plugin
   if (!includes(["ready", "outdated"], status.state)) {
@@ -88,7 +88,7 @@ export async function runContainerModule(
 ): Promise<RunResult> {
   const provider = <KubernetesProvider>ctx.provider
   const context = provider.config.context
-  const namespace = await getAppNamespace(ctx, provider)
+  const namespace = await getAppNamespace(ctx, log, provider)
   const image = await containerHelpers.getDeploymentImageId(module, provider.config.deploymentRegistry)
 
   return runPod({
@@ -126,7 +126,7 @@ export async function runContainerTask(
 
   const provider = <KubernetesProvider>ctx.provider
   const context = provider.config.context
-  const namespace = await getAppNamespace(ctx, provider)
+  const namespace = await getAppNamespace(ctx, log, provider)
   const image = await containerHelpers.getDeploymentImageId(module, provider.config.deploymentRegistry)
 
   const res = await runPod({
@@ -148,6 +148,7 @@ export async function runContainerTask(
 
   await storeTaskResult({
     ctx,
+    log,
     result,
     taskVersion,
     taskName: task.name,

garden-service/src/plugins/kubernetes/container/status.ts

@@ -29,11 +29,11 @@ export async function getContainerServiceStatus(
   // TODO: hash and compare all the configuration files (otherwise internal changes don't get deployed)
   const version = module.version
   const provider = k8sCtx.provider
-  const api = new KubeApi(provider.config.context)
-  const namespace = await getAppNamespace(k8sCtx, provider)
+  const api = await KubeApi.factory(log, provider.config.context)
+  const namespace = await getAppNamespace(k8sCtx, log, provider)
 
   // FIXME: [objects, matched] and ingresses can be run in parallel
-  const objects = await createContainerObjects(k8sCtx, service, runtimeContext, hotReload)
+  const objects = await createContainerObjects(k8sCtx, log, service, runtimeContext, hotReload)
   const { state, remoteObjects } = await compareDeployedObjects(k8sCtx, api, namespace, objects, log, true)
   const ingresses = await getIngresses(service, api, provider)
 

garden-service/src/plugins/kubernetes/container/test.ts

@@ -32,5 +32,5 @@ export async function testContainerModule(
     log,
   })
 
-  return storeTestResult({ ctx, module, testName, testVersion, result })
+  return storeTestResult({ ctx, log, module, testName, testVersion, result })
 }

garden-service/src/plugins/kubernetes/helm/build.ts

@@ -20,7 +20,7 @@ import { KubernetesPluginContext } from "../kubernetes"
 
 export async function buildHelmModule({ ctx, module, log }: BuildModuleParams<HelmModule>): Promise<BuildResult> {
   const k8sCtx = <KubernetesPluginContext>ctx
-  const namespace = await getNamespace({ ctx: k8sCtx, provider: k8sCtx.provider, skipCreate: true })
+  const namespace = await getNamespace({ ctx: k8sCtx, log, provider: k8sCtx.provider, skipCreate: true })
   const context = ctx.provider.config.context
   const baseModule = getBaseModule(module)
 

garden-service/src/plugins/kubernetes/helm/common.ts

@@ -42,7 +42,7 @@ export async function getChartResources(ctx: PluginContext, module: Module, log:
   const k8sCtx = <KubernetesPluginContext>ctx
   const chartPath = await getChartPath(module)
   const valuesPath = getValuesPath(chartPath)
-  const namespace = await getNamespace({ ctx: k8sCtx, provider: k8sCtx.provider, skipCreate: true })
+  const namespace = await getNamespace({ ctx: k8sCtx, log, provider: k8sCtx.provider, skipCreate: true })
   const context = ctx.provider.config.context
   const releaseName = getReleaseName(module)
 
@@ -269,7 +269,7 @@ async function renderHelmTemplateString(
   const tempFilePath = join(chartPath, "templates", cryptoRandomString(16))
   const valuesPath = getValuesPath(chartPath)
   const k8sCtx = <KubernetesPluginContext>ctx
-  const namespace = await getNamespace({ ctx: k8sCtx, provider: k8sCtx.provider, skipCreate: true })
+  const namespace = await getNamespace({ ctx: k8sCtx, log, provider: k8sCtx.provider, skipCreate: true })
   const releaseName = getReleaseName(module)
   const context = ctx.provider.config.context
 

garden-service/src/plugins/kubernetes/helm/deployment.ts

@@ -45,7 +45,7 @@ export async function deployService(
   const provider = k8sCtx.provider
   const chartPath = await getChartPath(module)
   const valuesPath = getValuesPath(chartPath)
-  const namespace = await getAppNamespace(k8sCtx, provider)
+  const namespace = await getAppNamespace(k8sCtx, log, provider)
   const context = provider.config.context
   const releaseName = getReleaseName(module)
 
@@ -103,7 +103,7 @@ export async function deleteService(params: DeleteServiceParams): Promise<Servic
   const { ctx, log, module } = params
 
   const k8sCtx = <KubernetesPluginContext>ctx
-  const namespace = await getAppNamespace(k8sCtx, k8sCtx.provider)
+  const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)
   const context = k8sCtx.provider.config.context
   const releaseName = getReleaseName(module)
 

garden-service/src/plugins/kubernetes/helm/logs.ts

@@ -17,7 +17,7 @@ export async function getServiceLogs(params: GetServiceLogsParams<HelmModule>) {
   const { ctx, module, log } = params
   const k8sCtx = <KubernetesPluginContext>ctx
   const context = k8sCtx.provider.config.context
-  const namespace = await getAppNamespace(k8sCtx, k8sCtx.provider)
+  const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)
 
   const resources = await getChartResources(k8sCtx, module, log)
 

garden-service/src/plugins/kubernetes/helm/run.ts

@@ -25,7 +25,7 @@ export async function runHelmModule(
 ): Promise<RunResult> {
   const k8sCtx = <KubernetesPluginContext>ctx
   const context = k8sCtx.provider.config.context
-  const namespace = await getAppNamespace(k8sCtx, k8sCtx.provider)
+  const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)
   const serviceResourceSpec = getServiceResourceSpec(module)
 
   if (!serviceResourceSpec) {
@@ -57,7 +57,7 @@ export async function runHelmTask(
 ): Promise<RunTaskResult> {
   const k8sCtx = <KubernetesPluginContext>ctx
   const context = k8sCtx.provider.config.context
-  const namespace = await getAppNamespace(k8sCtx, k8sCtx.provider)
+  const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)
 
   const args = task.spec.args
   const image = await getImage(k8sCtx, module, log, task.spec.resource || getServiceResourceSpec(module))
@@ -79,6 +79,7 @@ export async function runHelmTask(
 
   await storeTaskResult({
     ctx,
+    log,
     result,
     taskVersion,
     taskName: task.name,

garden-service/src/plugins/kubernetes/helm/status.ts

@@ -61,8 +61,8 @@ export async function getServiceStatus(
   }
 
   const provider = k8sCtx.provider
-  const api = new KubeApi(provider.config.context)
-  const namespace = await getAppNamespace(k8sCtx, provider)
+  const api = await KubeApi.factory(log, provider.config.context)
+  const namespace = await getAppNamespace(k8sCtx, log, provider)
   let { state, remoteObjects } = await compareDeployedObjects(k8sCtx, api, namespace, chartResources, log, false)
   const detail = { remoteObjects }
 

garden-service/src/plugins/kubernetes/helm/test.ts

@@ -27,7 +27,7 @@ export async function testHelmModule(
 
   const k8sCtx = <KubernetesPluginContext>ctx
   const context = k8sCtx.provider.config.context
-  const namespace = await getAppNamespace(k8sCtx, k8sCtx.provider)
+  const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)
 
   const chartResources = await getChartResources(k8sCtx, module, log)
   const resourceSpec = testConfig.spec.resource || getServiceResourceSpec(module)
@@ -48,5 +48,5 @@ export async function testHelmModule(
     log,
   })
 
-  return storeTestResult({ ctx: k8sCtx, module, testName, testVersion, result })
+  return storeTestResult({ ctx: k8sCtx, log, module, testName, testVersion, result })
 }