diff --git a/garden-service/src/plugins/kubernetes/constants.ts b/garden-service/src/plugins/kubernetes/constants.ts index 1cc6931604..efb5674a48 100644 --- a/garden-service/src/plugins/kubernetes/constants.ts +++ b/garden-service/src/plugins/kubernetes/constants.ts @@ -10,7 +10,9 @@ export const RSYNC_PORT = 873 export const CLUSTER_REGISTRY_PORT = 5000 export const CLUSTER_REGISTRY_DEPLOYMENT_NAME = "garden-docker-registry" export const MAX_CONFIGMAP_DATA_SIZE = 1024 * 1024 // max ConfigMap data size is 1MB -export const MAX_RUN_RESULT_OUTPUT_LENGTH = 900 * 1024 // max ConfigMap data size is 1MB, so 900kB gives enough margin +// max ConfigMap data size is 1MB but we need to factor in overhead, plus in some cases the log is duplicated in +// the outputs field, so we cap at 250kB. +export const MAX_RUN_RESULT_LOG_LENGTH = 250 * 1024 export const dockerAuthSecretName = "builder-docker-config" export const dockerAuthSecretKey = ".dockerconfigjson" diff --git a/garden-service/src/plugins/kubernetes/helm/common.ts b/garden-service/src/plugins/kubernetes/helm/common.ts index b47c4d45eb..89bd20dcf2 100644 --- a/garden-service/src/plugins/kubernetes/helm/common.ts +++ b/garden-service/src/plugins/kubernetes/helm/common.ts @@ -24,7 +24,7 @@ import { deline, tailString } from "../../../util/string" import { getAnnotation, flattenResources } from "../util" import { KubernetesPluginContext } from "../config" import { RunResult } from "../../../types/plugin/base" -import { MAX_RUN_RESULT_OUTPUT_LENGTH } from "../constants" +import { MAX_RUN_RESULT_LOG_LENGTH } from "../constants" const gardenValuesFilename = "garden-values.yml" @@ -261,7 +261,7 @@ export function loadTemplate(template: string) { } export function trimRunOutput(result: T): T { - const log = tailString(result.log, MAX_RUN_RESULT_OUTPUT_LENGTH, true) + const log = tailString(result.log, MAX_RUN_RESULT_LOG_LENGTH, true) return { ...result, diff --git a/garden-service/src/plugins/kubernetes/task-results.ts b/garden-service/src/plugins/kubernetes/task-results.ts index bec4ddb33d..ce7344232f 100644 --- a/garden-service/src/plugins/kubernetes/task-results.ts +++ b/garden-service/src/plugins/kubernetes/task-results.ts @@ -18,11 +18,13 @@ import { RunTaskResult } from "../../types/plugin/task/runTask" import { deserializeValues } from "../../util/util" import { PluginContext } from "../../plugin-context" import { LogEntry } from "../../logger/log-entry" -import { gardenAnnotationKey } from "../../util/string" +import { gardenAnnotationKey, tailString } from "../../util/string" import { Module } from "../../types/module" import hasha from "hasha" import { upsertConfigMap } from "./util" import { trimRunOutput } from "./helm/common" +import { MAX_RUN_RESULT_LOG_LENGTH } from "./constants" +import chalk from "chalk" export async function getTaskResult({ ctx, @@ -95,20 +97,29 @@ export async function storeTaskResult({ const api = await KubeApi.factory(log, provider) const namespace = await getMetadataNamespace(ctx, log, provider) + // FIXME: We should store the logs separately, because of the 1MB size limit on ConfigMaps. const data: RunTaskResult = trimRunOutput(result) - await upsertConfigMap({ - api, - namespace, - key: getTaskResultKey(ctx, module, taskName, taskVersion), - labels: { - [gardenAnnotationKey("module")]: module.name, - [gardenAnnotationKey("task")]: taskName, - [gardenAnnotationKey("moduleVersion")]: module.version.versionString, - [gardenAnnotationKey("version")]: taskVersion.versionString, - }, - data, - }) + if (data.outputs.log && typeof data.outputs.log === "string") { + data.outputs.log = tailString(data.outputs.log, MAX_RUN_RESULT_LOG_LENGTH, true) + } + + try { + await upsertConfigMap({ + api, + namespace, + key: getTaskResultKey(ctx, module, taskName, taskVersion), + labels: { + [gardenAnnotationKey("module")]: module.name, + [gardenAnnotationKey("task")]: taskName, + [gardenAnnotationKey("moduleVersion")]: module.version.versionString, + [gardenAnnotationKey("version")]: taskVersion.versionString, + }, + data, + }) + } catch (err) { + log.warn(chalk.yellow(`Unable to store task result: ${err.message}`)) + } return data } diff --git a/garden-service/src/plugins/kubernetes/test-results.ts b/garden-service/src/plugins/kubernetes/test-results.ts index f987c1c99f..28ed9b74c6 100644 --- a/garden-service/src/plugins/kubernetes/test-results.ts +++ b/garden-service/src/plugins/kubernetes/test-results.ts @@ -22,6 +22,7 @@ import { gardenAnnotationKey } from "../../util/string" import { upsertConfigMap } from "./util" import { trimRunOutput } from "./helm/common" import { getSystemNamespace } from "./namespace" +import chalk from "chalk" export async function getTestResult({ ctx, @@ -97,18 +98,22 @@ export async function storeTestResult({ const data: TestResult = trimRunOutput(result) - await upsertConfigMap({ - api, - namespace: testResultNamespace, - key: getTestResultKey(k8sCtx, module, testName, testVersion), - labels: { - [gardenAnnotationKey("module")]: module.name, - [gardenAnnotationKey("test")]: testName, - [gardenAnnotationKey("moduleVersion")]: module.version.versionString, - [gardenAnnotationKey("version")]: testVersion.versionString, - }, - data, - }) + try { + await upsertConfigMap({ + api, + namespace: testResultNamespace, + key: getTestResultKey(k8sCtx, module, testName, testVersion), + labels: { + [gardenAnnotationKey("module")]: module.name, + [gardenAnnotationKey("test")]: testName, + [gardenAnnotationKey("moduleVersion")]: module.version.versionString, + [gardenAnnotationKey("version")]: testVersion.versionString, + }, + data, + }) + } catch (err) { + log.warn(chalk.yellow(`Unable to store test result: ${err.message}`)) + } return data } diff --git a/garden-service/test/integ/src/plugins/kubernetes/task-results.ts b/garden-service/test/integ/src/plugins/kubernetes/task-results.ts new file mode 100644 index 0000000000..bc662d4758 --- /dev/null +++ b/garden-service/test/integ/src/plugins/kubernetes/task-results.ts @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { Garden } from "../../../../../src/garden" +import { Provider } from "../../../../../src/config/provider" +import { KubernetesConfig } from "../../../../../src/plugins/kubernetes/config" +import { getDataDir, makeTestGarden } from "../../../../helpers" +import { randomString } from "../../../../../src/util/string" +import { expect } from "chai" +import { storeTaskResult, getTaskResult } from "../../../../../src/plugins/kubernetes/task-results" +import { MAX_RUN_RESULT_LOG_LENGTH } from "../../../../../src/plugins/kubernetes/constants" + +describe("kubernetes task results", () => { + let garden: Garden + let provider: Provider + + before(async () => { + const root = getDataDir("test-projects", "container") + garden = await makeTestGarden(root) + provider = (await garden.resolveProvider("local-kubernetes")) as Provider + }) + + after(async () => { + await garden.close() + }) + + describe("storeTaskResult", () => { + it("should trim logs when necessary", async () => { + const ctx = garden.getPluginContext(provider) + const graph = await garden.getConfigGraph(garden.log) + const task = await graph.getTask("echo-task") + + const data = randomString(1024 * 1024) + + const trimmed = await storeTaskResult({ + ctx, + log: garden.log, + module: task.module, + taskName: task.name, + taskVersion: task.module.version, + result: { + moduleName: task.module.name, + taskName: task.name, + outputs: { log: data }, + log: data, + startedAt: new Date(), + completedAt: new Date(), + command: [], + version: task.module.version.versionString, + success: true, + }, + }) + + expect(trimmed.log.length).to.be.lte(MAX_RUN_RESULT_LOG_LENGTH) + + const stored = await getTaskResult({ + ctx, + log: garden.log, + module: task.module, + task, + taskVersion: task.module.version, + }) + + expect(stored).to.exist + expect(stored!.log.length).to.equal(trimmed.log.length) + + const outputsLog = stored!.outputs.log as string + expect(outputsLog.length).to.equal(trimmed.log.length) + }) + }) +})