feat(k8s): cache task results

Added caching for task results. Similarly to the caching for test runs
that was already in place, these are stored by version as ConfigMap-s.

The version used to index these results includes the task's module's
current version, along with the module versions of the task's
runtime dependencies (if any). This is the same formula that is used for
computing test task versions.

Currently, tasks always execute, regardless of whether they have cached
results for the current version, since we need to finish other changes
that will provide dependency results to the process methods of task
classes before the caching semantics can be finalized for tasks, in
relation to dependency calculations.

A versioning bug was also fixed for the test result caching
functionality. Here, the version used was just the test config's
module's version, whereas now the corresponding test task's version
is used to index it instead (the same as is now also used to cache
task results).

garden-service/src/actions.ts

@@ -37,9 +37,9 @@ import {
   TestResult,
   PluginActionOutputs,
   PublishResult,
-  RunTaskResult,
   TaskActionOutputs,
   HotReloadServiceResult,
+  RunTaskResult,
 } from "./types/plugin/outputs"
 import {
   BuildModuleParams,
@@ -70,6 +70,7 @@ import {
   PluginTaskActionParamsBase,
   RunTaskParams,
   TaskActionParams,
+  GetTaskResultParams,
 } from "./types/plugin/params"
 import { Service, ServiceStatus, getServiceRuntimeContext } from "./types/service"
 import { mapValues, values, keyBy, omit, pickBy, fromPairs } from "lodash"
@@ -329,6 +330,14 @@ export class ActionHelper implements TypeGuard {
     return this.callTaskHandler({ params, actionType: "runTask" })
   }
 
+  async getTaskResult(params: TaskActionHelperParams<GetTaskResultParams>): Promise<RunTaskResult | null> {
+    return this.callTaskHandler({
+      params,
+      actionType: "getTaskResult",
+      defaultHandler: async () => null,
+    })
+  }
+
   //endregion
 
   //===========================================================================

garden-service/src/commands/run/task.ts

@@ -59,7 +59,7 @@ export class RunTaskCommand extends Command<Args, Opts> {
 
     await garden.actions.prepareEnvironment({ log })
 
-    const taskTask = new TaskTask({ garden, graph, task, log, force: true, forceBuild: opts["force-build"] })
+    const taskTask = await TaskTask.factory({ garden, graph, task, log, force: true, forceBuild: opts["force-build"] })
     await garden.addTask(taskTask)
 
     const result = (await garden.processTasks())[taskTask.getBaseKey()]

garden-service/src/commands/run/test.ts

@@ -25,6 +25,7 @@ import dedent = require("dedent")
 import { prepareRuntimeContext } from "../../types/service"
 import { logHeader } from "../../logger/util"
 import { PushTask } from "../../tasks/push"
+import { getTestVersion } from "../../tasks/test"
 
 const runArgs = {
   module: new StringParameter({
@@ -101,13 +102,16 @@ export class RunTestCommand extends Command<Args, Opts> {
 
     printRuntimeContext(log, runtimeContext)
 
+    const testVersion = await getTestVersion(garden, graph, module, testConfig)
+
     const result = await garden.actions.testModule({
       log,
       module,
       interactive,
       runtimeContext,
       silent: false,
       testConfig,
+      testVersion,
     })
 
     return { result }

garden-service/src/plugins/kubernetes/container/handlers.ts

@@ -19,6 +19,7 @@ import { getContainerServiceStatus } from "./status"
 import { getTestResult } from "../test"
 import { ContainerModule } from "../../container/config"
 import { configureMavenContainerModule, MavenContainerModule } from "../../maven-container/maven-container"
+import { getTaskResult } from "../task-results"
 
 async function configure(params: ConfigureModuleParams<ContainerModule>) {
   const config = await configureContainerModule(params)
@@ -46,6 +47,7 @@ export const containerHandlers = {
   runModule: runContainerModule,
   runService: runContainerService,
   runTask: runContainerTask,
+  getTaskResult,
   testModule: testContainerModule,
 }
 

garden-service/src/plugins/kubernetes/container/run.ts

@@ -23,6 +23,7 @@ import { getContainerServiceStatus } from "./status"
 import { runPod } from "../run"
 import { containerHelpers } from "../../container/helpers"
 import { KubernetesPluginContext, KubernetesProvider } from "../kubernetes"
+import { storeTaskResult } from "../task-results"
 
 export async function execInService(params: ExecInServiceParams<ContainerModule>) {
   const { ctx, service, command, interactive } = params
@@ -115,7 +116,7 @@ export async function runContainerService(
 }
 
 export async function runContainerTask(
-  { ctx, log, module, task, interactive, runtimeContext }: RunTaskParams<ContainerModule>,
+  { ctx, log, module, task, taskVersion, interactive, runtimeContext }: RunTaskParams<ContainerModule>,
 ): Promise<RunTaskResult> {
   extend(runtimeContext.envVars, task.spec.env || {})
 
@@ -124,7 +125,7 @@ export async function runContainerTask(
   const namespace = await getAppNamespace(ctx, provider)
   const image = await containerHelpers.getDeploymentImageId(module, provider.config.deploymentRegistry)
 
-  const result = await runPod({
+  const res = await runPod({
     context,
     namespace,
     module,
@@ -139,8 +140,14 @@ export async function runContainerTask(
     log,
   })
 
-  return {
-    ...result,
+  const result = { ...res, taskName: task.name }
+
+  await storeTaskResult({
+    ctx,
+    result,
+    taskVersion,
     taskName: task.name,
-  }
+  })
+
+  return result
 }

garden-service/src/plugins/kubernetes/container/test.ts

@@ -14,7 +14,7 @@ import { runContainerModule } from "./run"
 import { storeTestResult } from "../test"
 
 export async function testContainerModule(
-  { ctx, interactive, module, runtimeContext, testConfig, log }:
+  { ctx, interactive, module, runtimeContext, testConfig, testVersion, log }:
     TestModuleParams<ContainerModule>,
 ): Promise<TestResult> {
   const testName = testConfig.name
@@ -32,5 +32,5 @@ export async function testContainerModule(
     log,
   })
 
-  return storeTestResult({ ctx, module, testName, result })
+  return storeTestResult({ ctx, module, testName, testVersion, result })
 }

garden-service/src/plugins/kubernetes/helm/run.ts

@@ -16,6 +16,7 @@ import { PluginContext } from "../../../plugin-context"
 import { LogEntry } from "../../../logger/log-entry"
 import { ConfigurationError } from "../../../exceptions"
 import { KubernetesPluginContext } from "../kubernetes"
+import { storeTaskResult } from "../task-results"
 
 export async function runHelmModule(
   {
@@ -52,7 +53,7 @@ export async function runHelmModule(
 }
 
 export async function runHelmTask(
-  { ctx, log, module, task, interactive, runtimeContext, timeout }: RunTaskParams<HelmModule>,
+  { ctx, log, module, task, taskVersion, interactive, runtimeContext, timeout }: RunTaskParams<HelmModule>,
 ): Promise<RunTaskResult> {
   const k8sCtx = <KubernetesPluginContext>ctx
   const context = k8sCtx.provider.config.context
@@ -74,10 +75,16 @@ export async function runHelmTask(
     log,
   })
 
-  return {
+  const result = { ...res, taskName: task.name }
+
+  await storeTaskResult({
+    ctx,
+    result,
+    taskVersion,
     taskName: task.name,
-    ...res,
-  }
+  })
+
+  return result
 }
 
 async function getImage(ctx: PluginContext, module: HelmModule, log: LogEntry, resourceSpec: HelmResourceSpec) {

garden-service/src/plugins/kubernetes/helm/test.ts

@@ -17,7 +17,7 @@ import { findServiceResource, getChartResources, getResourceContainer, getServic
 import { KubernetesPluginContext } from "../kubernetes"
 
 export async function testHelmModule(
-  { ctx, log, interactive, module, runtimeContext, testConfig }:
+  { ctx, log, interactive, module, runtimeContext, testConfig, testVersion }:
     TestModuleParams<HelmModule>,
 ): Promise<TestResult> {
   const testName = testConfig.name
@@ -48,5 +48,5 @@ export async function testHelmModule(
     log,
   })
 
-  return storeTestResult({ ctx: k8sCtx, module, testName, result })
+  return storeTestResult({ ctx: k8sCtx, module, testName, testVersion, result })
 }

garden-service/src/plugins/kubernetes/task-results.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2018 Garden Technologies, Inc. <[email protected]>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import { GetTaskResultParams } from "../../types/plugin/params"
+import { ContainerModule } from "../container/config"
+import { HelmModule } from "./helm/config"
+import { ModuleVersion } from "../../vcs/base"
+import { KubernetesPluginContext, KubernetesProvider } from "./kubernetes"
+import { KubeApi } from "./api"
+import { getMetadataNamespace } from "./namespace"
+import { RunTaskResult } from "../../types/plugin/outputs"
+import { deserializeValues, serializeValues } from "../../util/util"
+import { PluginContext } from "../../plugin-context"
+
+export async function getTaskResult(
+  { ctx, task, taskVersion }: GetTaskResultParams<ContainerModule | HelmModule>,
+): Promise<RunTaskResult | null> {
+  const k8sCtx = <KubernetesPluginContext>ctx
+  const api = new KubeApi(k8sCtx.provider.config.context)
+  const ns = await getMetadataNamespace(k8sCtx, k8sCtx.provider)
+  const resultKey = getTaskResultKey(task.name, taskVersion)
+
+  try {
+    const res = await api.core.readNamespacedConfigMap(resultKey, ns)
+    return <RunTaskResult>deserializeValues(res.body.data)
+  } catch (err) {
+    if (err.code === 404) {
+      return null
+    } else {
+      throw err
+    }
+  }
+}
+
+export function getTaskResultKey(taskName: string, version: ModuleVersion) {
+  return `task-result--${taskName}--${version.versionString}`
+}
+
+/**
+ * Store a task run result as a ConfigMap in the cluster.
+ *
+ * TODO: Implement a CRD for this.
+ */
+export async function storeTaskResult(
+  { ctx, taskName, taskVersion, result }:
+    { ctx: PluginContext, taskName: string, taskVersion: ModuleVersion, result: RunTaskResult },
+): Promise<RunTaskResult> {
+  const provider = <KubernetesProvider>ctx.provider
+  const api = new KubeApi(provider.config.context)
+  const ns = await getMetadataNamespace(ctx, provider)
+  const resultKey = getTaskResultKey(taskName, taskVersion)
+
+  const body = {
+    apiVersion: "v1",
+    kind: "ConfigMap",
+    metadata: {
+      name: resultKey,
+      annotations: {
+        "garden.io/generated": "true",
+      },
+    },
+    data: serializeValues(result),
+  }
+
+  try {
+    await api.core.createNamespacedConfigMap(ns, <any>body)
+  } catch (err) {
+    if (err.code === 409) {
+      await api.core.patchNamespacedConfigMap(resultKey, ns, body)
+    } else {
+      throw err
+    }
+  }
+
+  return result
+}

garden-service/src/plugins/kubernetes/test.ts

@@ -19,12 +19,12 @@ import { PluginContext } from "../../plugin-context"
 import { KubernetesPluginContext } from "./kubernetes"
 
 export async function getTestResult(
-  { ctx, module, testName, version }: GetTestResultParams<ContainerModule | HelmModule>,
-) {
+  { ctx, module, testName, testVersion }: GetTestResultParams<ContainerModule | HelmModule>,
+): Promise<TestResult | null> {
   const k8sCtx = <KubernetesPluginContext>ctx
   const api = new KubeApi(k8sCtx.provider.config.context)
   const ns = await getMetadataNamespace(k8sCtx, k8sCtx.provider)
-  const resultKey = getTestResultKey(module, testName, version)
+  const resultKey = getTestResultKey(module, testName, testVersion)
 
   try {
     const res = await api.core.readNamespacedConfigMap(resultKey, ns)
@@ -48,9 +48,9 @@ export function getTestResultKey(module: Module, testName: string, version: Modu
  * TODO: Implement a CRD for this.
  */
 export async function storeTestResult(
-  { ctx, module, testName, result }:
-    { ctx: PluginContext, module: Module, testName: string, result: RunResult },
-) {
+  { ctx, module, testName, testVersion, result }:
+    { ctx: PluginContext, module: Module, testName: string, testVersion: ModuleVersion, result: RunResult },
+): Promise<TestResult> {
   const k8sCtx = <KubernetesPluginContext>ctx
   const api = new KubeApi(k8sCtx.provider.config.context)
 
@@ -60,7 +60,7 @@ export async function storeTestResult(
   }
 
   const ns = await getMetadataNamespace(k8sCtx, k8sCtx.provider)
-  const resultKey = getTestResultKey(module, testName, result.version)
+  const resultKey = getTestResultKey(module, testName, testVersion)
   const body = {
     apiVersion: "v1",
     kind: "ConfigMap",

garden-service/src/tasks/deploy.ts

@@ -72,13 +72,13 @@ export class DeployTask extends BaseTask {
     if (this.fromWatch && includes(this.hotReloadServiceNames, this.service.name)) {
       return deployTasks
     } else {
-      const taskTasks = deps.task.map(task => {
-        return new TaskTask({
+      const taskTasks = await Bluebird.map(deps.task, (task) => {
+        return TaskTask.factory({
           task,
           garden: this.garden,
           log: this.log,
           graph: this.graph,
-          force: false,
+          force: this.force,
           forceBuild: this.forceBuild,
         })
       })