fix(k8s): enable publishing container modules when using remote builders

This has the caveat that you need to have a local Docker daemon running.
Avoiding that requirement would take much more work, since we'd need to
tackle all manner of authentication/key management issues.

garden-service/src/actions.ts

@@ -285,7 +285,7 @@ export class ActionHelper implements TypeGuard {
   async publishModule<T extends Module>(
     params: ModuleActionHelperParams<PublishModuleParams<T>>,
   ): Promise<PublishResult> {
-    return this.callModuleHandler({ params, actionType: "publishModule", defaultHandler: dummyPublishHandler })
+    return this.callModuleHandler({ params, actionType: "publish", defaultHandler: dummyPublishHandler })
   }
 
   async runModule<T extends Module>(params: ModuleActionHelperParams<RunModuleParams<T>>): Promise<RunResult> {

garden-service/src/plugins/container/container.ts

@@ -16,9 +16,9 @@ import { ContainerModule, containerModuleSpecSchema } from "./config"
 import { buildContainerModule, getContainerBuildStatus } from "./build"
 import { KubernetesProvider } from "../kubernetes/config"
 import { ConfigureModuleParams } from "../../types/plugin/module/configure"
-import { PublishModuleParams } from "../../types/plugin/module/publishModule"
 import { HotReloadServiceParams } from "../../types/plugin/service/hotReloadService"
 import { joi } from "../../config/common"
+import { publishContainerModule } from "./publish"
 
 export const containerModuleOutputsSchema = joi.object()
   .keys({
@@ -149,29 +149,7 @@ export const gardenPlugin = (): GardenPlugin => ({
       configure: configureContainerModule,
       getBuildStatus: getContainerBuildStatus,
       build: buildContainerModule,
-
-      async publishModule({ module, log }: PublishModuleParams<ContainerModule>) {
-        if (!(await containerHelpers.hasDockerfile(module))) {
-          log.setState({ msg: `Nothing to publish` })
-          return { published: false }
-        }
-
-        const localId = await containerHelpers.getLocalImageId(module)
-        const remoteId = await containerHelpers.getPublicImageId(module)
-
-        log.setState({ msg: `Publishing image ${remoteId}...` })
-
-        if (localId !== remoteId) {
-          await containerHelpers.dockerCli(module, ["tag", localId, remoteId])
-        }
-
-        // TODO: log error if it occurs
-        // TODO: stream output to log if at debug log level
-        // TODO: check if module already exists remotely?
-        await containerHelpers.dockerCli(module, ["push", remoteId])
-
-        return { published: true, message: `Published ${remoteId}` }
-      },
+      publish: publishContainerModule,
 
       async hotReloadService(_: HotReloadServiceParams) {
         return {}

garden-service/src/plugins/container/publish.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2018 Garden Technologies, Inc. <[email protected]>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import { ContainerModule } from "./config"
+import { PublishModuleParams } from "../../types/plugin/module/publishModule"
+import { containerHelpers } from "./helpers"
+
+export async function publishContainerModule({ module, log }: PublishModuleParams<ContainerModule>) {
+  if (!(await containerHelpers.hasDockerfile(module))) {
+    log.setState({ msg: `Nothing to publish` })
+    return { published: false }
+  }
+
+  const localId = await containerHelpers.getLocalImageId(module)
+  const remoteId = await containerHelpers.getPublicImageId(module)
+
+  log.setState({ msg: `Publishing image ${remoteId}...` })
+
+  if (localId !== remoteId) {
+    await containerHelpers.dockerCli(module, ["tag", localId, remoteId])
+  }
+
+  // TODO: stream output to log if at debug log level
+  await containerHelpers.dockerCli(module, ["push", remoteId])
+
+  return { published: true, message: `Published ${remoteId}` }
+}

garden-service/src/plugins/kubernetes/constants.ts

@@ -7,3 +7,5 @@
  */
 
 export const RSYNC_PORT = 873
+export const CLUSTER_REGISTRY_PORT = 5000
+export const CLUSTER_REGISTRY_DEPLOYMENT_NAME = "garden-docker-registry"

garden-service/src/plugins/kubernetes/container/build.ts

@@ -20,19 +20,18 @@ import { posix, resolve } from "path"
 import { KubeApi } from "../api"
 import { kubectl } from "../kubectl"
 import { LogEntry } from "../../../logger/log-entry"
-import { KubernetesProvider, ContainerBuildMode } from "../config"
+import { KubernetesProvider, ContainerBuildMode, KubernetesPluginContext } from "../config"
 import { PluginError } from "../../../exceptions"
-import axios from "axios"
 import { runPod } from "../run"
 import { getRegistryHostname } from "../init"
+import { getManifestFromRegistry } from "./util"
 
 const dockerDaemonDeploymentName = "garden-docker-daemon"
 const dockerDaemonContainerName = "docker-daemon"
 // TODO: make build timeout configurable
 const buildTimeout = 600
 // Note: v0.9.0 appears to be completely broken: https://github.com/GoogleContainerTools/kaniko/issues/268
 const kanikoImage = "gcr.io/kaniko-project/executor:v0.8.0"
-const registryDeploymentName = "garden-docker-registry"
 const registryPort = 5000
 const syncDataVolumeName = "garden-build-sync"
 const syncDeploymentName = "garden-build-sync"
@@ -78,39 +77,10 @@ const getLocalBuildStatus: BuildStatusHandler = async (params) => {
 
 const getRemoteBuildStatus: BuildStatusHandler = async (params) => {
   const { ctx, module, log } = params
-  const provider = <KubernetesProvider>ctx.provider
-
-  const registryFwd = await getPortForward({
-    ctx,
-    log,
-    namespace: systemNamespace,
-    targetDeployment: `Deployment/${registryDeploymentName}`,
-    port: registryPort,
-  })
+  const k8sCtx = ctx as KubernetesPluginContext
+  const manifest = await getManifestFromRegistry(k8sCtx, module, log)
 
-  const imageId = await containerHelpers.getDeploymentImageId(module, provider.config.deploymentRegistry)
-  const imageName = containerHelpers.unparseImageId({
-    ...containerHelpers.parseImageId(imageId),
-    host: undefined,
-    tag: undefined,
-  })
-
-  const url = `http://localhost:${registryFwd.localPort}/v2/${imageName}/manifests/${module.version.versionString}`
-
-  try {
-    const res = await axios({ url })
-    log.silly(res.data)
-    return { ready: true }
-  } catch (err) {
-    if (err.response && err.response.status === 404) {
-      return { ready: false }
-    } else {
-      throw new PluginError(`Could not query in-cluster registry: ${err}`, {
-        message: err.message,
-        response: err.response,
-      })
-    }
-  }
+  return { ready: !!manifest }
 }
 
 const buildStatusHandlers: { [mode in ContainerBuildMode]: BuildStatusHandler } = {

garden-service/src/plugins/kubernetes/container/handlers.ts

@@ -21,6 +21,7 @@ import { ContainerModule } from "../../container/config"
 import { configureMavenContainerModule, MavenContainerModule } from "../../maven-container/maven-container"
 import { getTaskResult } from "../task-results"
 import { k8sBuildContainer, k8sGetContainerBuildStatus } from "./build"
+import { k8sPublishContainerModule } from "./publish"
 
 async function configure(params: ConfigureModuleParams<ContainerModule>) {
   params.moduleConfig = await configureContainerModule(params)
@@ -44,6 +45,7 @@ export const containerHandlers = {
   getServiceStatus: getContainerServiceStatus,
   getTestResult,
   hotReloadService: hotReloadContainer,
+  publish: k8sPublishContainerModule,
   runModule: runContainerModule,
   runService: runContainerService,
   runTask: runContainerTask,

garden-service/src/plugins/kubernetes/container/publish.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2018 Garden Technologies, Inc. <[email protected]>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import { ContainerModule } from "../../container/config"
+import { PublishModuleParams } from "../../../types/plugin/module/publishModule"
+import { containerHelpers } from "../../container/helpers"
+import { KubernetesPluginContext } from "../config"
+import { publishContainerModule } from "../../container/publish"
+import { getRegistryPortForward } from "./util"
+import execa = require("execa")
+
+export async function k8sPublishContainerModule(params: PublishModuleParams<ContainerModule>) {
+  const { ctx, module, log } = params
+  const k8sCtx = ctx as KubernetesPluginContext
+  const provider = k8sCtx.provider
+
+  if (!(await containerHelpers.hasDockerfile(module))) {
+    log.setState({ msg: `Nothing to publish` })
+    return { published: false }
+  }
+
+  if (provider.config.buildMode !== "local-docker") {
+    // First pull from the in-cluster registry, then resume standard publish flow.
+    // This does mean we require a local docker as a go-between, but the upside is that we can rely on the user's
+    // standard authentication setup, instead of having to re-implement or account for all the different ways the
+    // user might be authenticating with their registries.
+    log.setState(`Pulling from cluster container registry...`)
+
+    const fwd = await getRegistryPortForward(k8sCtx, log)
+
+    const imageId = await containerHelpers.getDeploymentImageId(module, ctx.provider.config.deploymentRegistry)
+    const pullImageName = containerHelpers.unparseImageId({
+      ...containerHelpers.parseImageId(imageId),
+      // Note: using localhost directly here has issues with Docker for Mac.
+      // https://github.com/docker/for-mac/issues/3611
+      host: `local.app.garden:${fwd.localPort}`,
+    })
+
+    await execa("docker", ["pull", pullImageName])
+  }
+
+  return publishContainerModule(params)
+}

garden-service/src/plugins/kubernetes/container/util.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2018 Garden Technologies, Inc. <[email protected]>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import { resolve } from "url"
+import { ContainerModule } from "../../container/config"
+import { getPortForward } from "../util"
+import { systemNamespace } from "../system"
+import { CLUSTER_REGISTRY_DEPLOYMENT_NAME, CLUSTER_REGISTRY_PORT } from "../constants"
+import { containerHelpers } from "../../container/helpers"
+import { PluginError } from "../../../exceptions"
+import { PluginContext } from "../../../plugin-context"
+import { LogEntry } from "../../../logger/log-entry"
+import { KubernetesPluginContext } from "../config"
+import axios from "axios"
+
+export async function getRegistryPortForward(ctx: PluginContext, log: LogEntry) {
+  return getPortForward({
+    ctx,
+    log,
+    namespace: systemNamespace,
+    targetDeployment: `Deployment/${CLUSTER_REGISTRY_DEPLOYMENT_NAME}`,
+    port: CLUSTER_REGISTRY_PORT,
+  })
+}
+
+export async function getManifestFromRegistry(
+  ctx: KubernetesPluginContext, module: ContainerModule, log: LogEntry,
+) {
+  const url = await getImageRegistryUrl(ctx, module, log, `manifests/${module.version.versionString}`)
+
+  try {
+    const res = await axios({ url })
+    log.silly(res.data)
+    return res.data
+  } catch (err) {
+    if (err.response && err.response.status === 404) {
+      return null
+    } else {
+      throw new PluginError(`Could not query in-cluster registry: ${err}`, {
+        message: err.message,
+        response: err.response,
+      })
+    }
+  }
+}
+
+async function getImageRegistryUrl(ctx: KubernetesPluginContext, module: ContainerModule, log: LogEntry, path: string) {
+  const registryFwd = await getRegistryPortForward(ctx, log)
+  const imageId = await containerHelpers.getDeploymentImageId(module, ctx.provider.config.deploymentRegistry)
+  const imageName = containerHelpers.unparseImageId({
+    ...containerHelpers.parseImageId(imageId),
+    host: undefined,
+    tag: undefined,
+  })
+
+  const baseUrl = `http://localhost:${registryFwd.localPort}/v2/${imageName}/`
+
+  return resolve(baseUrl, path)
+}

garden-service/src/types/plugin/outputs.ts

@@ -343,7 +343,7 @@ export interface ModuleActionOutputs extends ServiceActionOutputs {
   configure: Promise<ConfigureModuleResult>
   getBuildStatus: Promise<BuildStatus>
   build: Promise<BuildResult>
-  publishModule: Promise<PublishResult>
+  publish: Promise<PublishResult>
   runModule: Promise<RunResult>
   testModule: Promise<TestResult>
   getTestResult: Promise<TestResult | null>

garden-service/src/types/plugin/params.ts

@@ -369,7 +369,7 @@ export interface ModuleActionParams<T extends Module = Module> {
   configure: ConfigureModuleParams<T>
   getBuildStatus: GetBuildStatusParams<T>
   build: BuildModuleParams<T>
-  publishModule: PublishModuleParams<T>
+  publish: PublishModuleParams<T>
   runModule: RunModuleParams<T>
   testModule: TestModuleParams<T>
   getTestResult: GetTestResultParams<T>

garden-service/src/types/plugin/plugin.ts

@@ -163,7 +163,7 @@ export interface ModuleActionParams<T extends Module = Module> {
   configure: ConfigureModuleParams<T>
   getBuildStatus: GetBuildStatusParams<T>
   build: BuildModuleParams<T>
-  publishModule: PublishModuleParams<T>
+  publish: PublishModuleParams<T>
   runModule: RunModuleParams<T>
   testModule: TestModuleParams<T>
   getTestResult: GetTestResultParams<T>
@@ -174,7 +174,7 @@ export interface ModuleActionOutputs extends ServiceActionOutputs {
   configure: Promise<ConfigureModuleResult>
   getBuildStatus: Promise<BuildStatus>
   build: Promise<BuildResult>
-  publishModule: Promise<PublishResult>
+  publish: Promise<PublishResult>
   runModule: Promise<RunResult>
   testModule: Promise<TestResult>
   getTestResult: Promise<TestResult | null>
@@ -186,7 +186,7 @@ export const moduleActionDescriptions:
   configure,
   getBuildStatus,
   build,
-  publishModule,
+  publish: publishModule,
   runModule,
   testModule,
   getTestResult,

garden-service/test/unit/src/actions.ts

@@ -467,8 +467,8 @@ const testPlugin: PluginFactory = async () => ({
         return {}
       },
 
-      publishModule: async (params) => {
-        validate(params, moduleActionDescriptions.publishModule.paramsSchema)
+      publish: async (params) => {
+        validate(params, moduleActionDescriptions.publish.paramsSchema)
         return { published: true }
       },
 

garden-service/test/unit/src/commands/publish.ts

@@ -29,7 +29,7 @@ const testProvider: PluginFactory = () => {
         configure: configureTestModule,
         getBuildStatus,
         build,
-        publishModule,
+        publish: publishModule,
       },
     },
   }

garden-service/test/unit/src/plugins/container.ts

@@ -29,7 +29,7 @@ describe("plugins.container", () => {
   const handler = gardenPlugin()
   const configure = handler.moduleActions!.container!.configure!
   const build = handler.moduleActions!.container!.build!
-  const publishModule = handler.moduleActions!.container!.publishModule!
+  const publishModule = handler.moduleActions!.container!.publish!
   const getBuildStatus = handler.moduleActions!.container!.getBuildStatus!
 
   const baseConfig: ModuleConfig<ContainerModuleSpec, any, any> = {