fix(k8s): error when task logs were longer than 500kB

This fix is a hacky temporary fix, just to get rid of the error case.
A better solution would be to store logs as files somehow, e.g. in a
persistent volume.

garden-service/src/plugins/kubernetes/constants.ts

@@ -10,7 +10,9 @@ export const RSYNC_PORT = 873
 export const CLUSTER_REGISTRY_PORT = 5000
 export const CLUSTER_REGISTRY_DEPLOYMENT_NAME = "garden-docker-registry"
 export const MAX_CONFIGMAP_DATA_SIZE = 1024 * 1024 // max ConfigMap data size is 1MB
-export const MAX_RUN_RESULT_OUTPUT_LENGTH = 900 * 1024 // max ConfigMap data size is 1MB, so 900kB gives enough margin
+// max ConfigMap data size is 1MB but we need to factor in overhead, plus in some cases the log is duplicated in
+// the outputs field, so we cap at 250kB.
+export const MAX_RUN_RESULT_LOG_LENGTH = 250 * 1024
 
 export const dockerAuthSecretName = "builder-docker-config"
 export const dockerAuthSecretKey = ".dockerconfigjson"

garden-service/src/plugins/kubernetes/helm/common.ts

@@ -24,7 +24,7 @@ import { deline, tailString } from "../../../util/string"
 import { getAnnotation, flattenResources } from "../util"
 import { KubernetesPluginContext } from "../config"
 import { RunResult } from "../../../types/plugin/base"
-import { MAX_RUN_RESULT_OUTPUT_LENGTH } from "../constants"
+import { MAX_RUN_RESULT_LOG_LENGTH } from "../constants"
 
 const gardenValuesFilename = "garden-values.yml"
 
@@ -261,7 +261,7 @@ export function loadTemplate(template: string) {
 }
 
 export function trimRunOutput<T extends RunResult>(result: T): T {
-  const log = tailString(result.log, MAX_RUN_RESULT_OUTPUT_LENGTH, true)
+  const log = tailString(result.log, MAX_RUN_RESULT_LOG_LENGTH, true)
 
   return {
     ...result,

garden-service/src/plugins/kubernetes/task-results.ts

@@ -18,11 +18,13 @@ import { RunTaskResult } from "../../types/plugin/task/runTask"
 import { deserializeValues } from "../../util/util"
 import { PluginContext } from "../../plugin-context"
 import { LogEntry } from "../../logger/log-entry"
-import { gardenAnnotationKey } from "../../util/string"
+import { gardenAnnotationKey, tailString } from "../../util/string"
 import { Module } from "../../types/module"
 import hasha from "hasha"
 import { upsertConfigMap } from "./util"
 import { trimRunOutput } from "./helm/common"
+import { MAX_RUN_RESULT_LOG_LENGTH } from "./constants"
+import chalk from "chalk"
 
 export async function getTaskResult({
   ctx,
@@ -95,20 +97,29 @@ export async function storeTaskResult({
   const api = await KubeApi.factory(log, provider)
   const namespace = await getMetadataNamespace(ctx, log, provider)
 
+  // FIXME: We should store the logs separately, because of the 1MB size limit on ConfigMaps.
   const data: RunTaskResult = trimRunOutput(result)
 
-  await upsertConfigMap({
-    api,
-    namespace,
-    key: getTaskResultKey(ctx, module, taskName, taskVersion),
-    labels: {
-      [gardenAnnotationKey("module")]: module.name,
-      [gardenAnnotationKey("task")]: taskName,
-      [gardenAnnotationKey("moduleVersion")]: module.version.versionString,
-      [gardenAnnotationKey("version")]: taskVersion.versionString,
-    },
-    data,
-  })
+  if (data.outputs.log && typeof data.outputs.log === "string") {
+    data.outputs.log = tailString(data.outputs.log, MAX_RUN_RESULT_LOG_LENGTH, true)
+  }
+
+  try {
+    await upsertConfigMap({
+      api,
+      namespace,
+      key: getTaskResultKey(ctx, module, taskName, taskVersion),
+      labels: {
+        [gardenAnnotationKey("module")]: module.name,
+        [gardenAnnotationKey("task")]: taskName,
+        [gardenAnnotationKey("moduleVersion")]: module.version.versionString,
+        [gardenAnnotationKey("version")]: taskVersion.versionString,
+      },
+      data,
+    })
+  } catch (err) {
+    log.warn(chalk.yellow(`Unable to store task result: ${err.message}`))
+  }
 
   return data
 }

garden-service/src/plugins/kubernetes/test-results.ts

@@ -22,6 +22,7 @@ import { gardenAnnotationKey } from "../../util/string"
 import { upsertConfigMap } from "./util"
 import { trimRunOutput } from "./helm/common"
 import { getSystemNamespace } from "./namespace"
+import chalk from "chalk"
 
 export async function getTestResult({
   ctx,
@@ -97,18 +98,22 @@ export async function storeTestResult({
 
   const data: TestResult = trimRunOutput(result)
 
-  await upsertConfigMap({
-    api,
-    namespace: testResultNamespace,
-    key: getTestResultKey(k8sCtx, module, testName, testVersion),
-    labels: {
-      [gardenAnnotationKey("module")]: module.name,
-      [gardenAnnotationKey("test")]: testName,
-      [gardenAnnotationKey("moduleVersion")]: module.version.versionString,
-      [gardenAnnotationKey("version")]: testVersion.versionString,
-    },
-    data,
-  })
+  try {
+    await upsertConfigMap({
+      api,
+      namespace: testResultNamespace,
+      key: getTestResultKey(k8sCtx, module, testName, testVersion),
+      labels: {
+        [gardenAnnotationKey("module")]: module.name,
+        [gardenAnnotationKey("test")]: testName,
+        [gardenAnnotationKey("moduleVersion")]: module.version.versionString,
+        [gardenAnnotationKey("version")]: testVersion.versionString,
+      },
+      data,
+    })
+  } catch (err) {
+    log.warn(chalk.yellow(`Unable to store test result: ${err.message}`))
+  }
 
   return data
 }

garden-service/test/integ/src/plugins/kubernetes/task-results.ts

@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2018-2020 Garden Technologies, Inc. <[email protected]>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import { Garden } from "../../../../../src/garden"
+import { Provider } from "../../../../../src/config/provider"
+import { KubernetesConfig } from "../../../../../src/plugins/kubernetes/config"
+import { getDataDir, makeTestGarden } from "../../../../helpers"
+import { randomString } from "../../../../../src/util/string"
+import { expect } from "chai"
+import { storeTaskResult, getTaskResult } from "../../../../../src/plugins/kubernetes/task-results"
+import { MAX_RUN_RESULT_LOG_LENGTH } from "../../../../../src/plugins/kubernetes/constants"
+
+describe("kubernetes task results", () => {
+  let garden: Garden
+  let provider: Provider<KubernetesConfig>
+
+  before(async () => {
+    const root = getDataDir("test-projects", "container")
+    garden = await makeTestGarden(root)
+    provider = (await garden.resolveProvider("local-kubernetes")) as Provider<KubernetesConfig>
+  })
+
+  after(async () => {
+    await garden.close()
+  })
+
+  describe("storeTaskResult", () => {
+    it("should trim logs when necessary", async () => {
+      const ctx = garden.getPluginContext(provider)
+      const graph = await garden.getConfigGraph(garden.log)
+      const task = await graph.getTask("echo-task")
+
+      const data = randomString(1024 * 1024)
+
+      const trimmed = await storeTaskResult({
+        ctx,
+        log: garden.log,
+        module: task.module,
+        taskName: task.name,
+        taskVersion: task.module.version,
+        result: {
+          moduleName: task.module.name,
+          taskName: task.name,
+          outputs: { log: data },
+          log: data,
+          startedAt: new Date(),
+          completedAt: new Date(),
+          command: [],
+          version: task.module.version.versionString,
+          success: true,
+        },
+      })
+
+      expect(trimmed.log.length).to.be.lte(MAX_RUN_RESULT_LOG_LENGTH)
+
+      const stored = await getTaskResult({
+        ctx,
+        log: garden.log,
+        module: task.module,
+        task,
+        taskVersion: task.module.version,
+      })
+
+      expect(stored).to.exist
+      expect(stored!.log.length).to.equal(trimmed.log.length)
+
+      const outputsLog = stored!.outputs.log as string
+      expect(outputsLog.length).to.equal(trimmed.log.length)
+    })
+  })
+})