diff --git a/docs/reference/module-types/container.md b/docs/reference/module-types/container.md index 393c537183..ac69e8c90e 100644 --- a/docs/reference/module-types/container.md +++ b/docs/reference/module-types/container.md @@ -13,8 +13,7 @@ guide](../../using-garden/configuration-files.md). The [first section](#configuration-keys) lists and describes the available schema keys. The [second section](#complete-yaml-schema) contains the complete YAML schema. -`container` modules also export values that are available in template strings under `${modules..outputs}`. -See the [Outputs](#outputs) section below for details. +`container` modules also export values that are available in template strings. See the [Outputs](#outputs) section below for details. ## Configuration keys @@ -1044,6 +1043,8 @@ tasks: ## Outputs +### Module outputs + The following keys are available via the `${modules.}` template string key for `container` modules. @@ -1132,3 +1133,18 @@ outputs: ... deployment-image-name: "my-deployment-registry.io/my-org/my-module" ``` + + +### Task outputs + +The following keys are available via the `${runtime.tasks.}` template string key for `container` module tasks. +Note that these are only resolved when deploying/running dependants of the task, so they are not usable for every field. + +### `runtime.tasks..outputs.log` + +The full log from the executed task. (Pro-tip: Make it machine readable so it can be parsed by dependant tasks and services!) + +| Type | Required | Default | +| -------- | -------- | ------- | +| `string` | No | `""` | + diff --git a/docs/reference/module-types/exec.md b/docs/reference/module-types/exec.md index 2758146cb9..b5a54d3e27 100644 --- a/docs/reference/module-types/exec.md +++ b/docs/reference/module-types/exec.md @@ -8,8 +8,7 @@ guide](../../using-garden/configuration-files.md). The [first section](#configuration-keys) lists and describes the available schema keys. The [second section](#complete-yaml-schema) contains the complete YAML schema. -`exec` modules also export values that are available in template strings under `${modules..outputs}`. -See the [Outputs](#outputs) section below for details. +`exec` modules also export values that are available in template strings. See the [Outputs](#outputs) section below for details. ## Configuration keys @@ -380,6 +379,8 @@ tests: ## Outputs +### Module outputs + The following keys are available via the `${modules.}` template string key for `exec` modules. @@ -432,3 +433,18 @@ The outputs defined by the module. | Type | Required | | -------- | -------- | | `object` | Yes | + + +### Task outputs + +The following keys are available via the `${runtime.tasks.}` template string key for `exec` module tasks. +Note that these are only resolved when deploying/running dependants of the task, so they are not usable for every field. + +### `runtime.tasks..outputs.log` + +The full log from the executed task. (Pro-tip: Make it machine readable so it can be parsed by dependant tasks and services!) + +| Type | Required | Default | +| -------- | -------- | ------- | +| `string` | No | `""` | + diff --git a/docs/reference/module-types/helm.md b/docs/reference/module-types/helm.md index cf41810e31..e21ba64e6e 100644 --- a/docs/reference/module-types/helm.md +++ b/docs/reference/module-types/helm.md @@ -8,8 +8,7 @@ guide](../../using-garden/configuration-files.md). The [first section](#configuration-keys) lists and describes the available schema keys. The [second section](#complete-yaml-schema) contains the complete YAML schema. -`helm` modules also export values that are available in template strings under `${modules..outputs}`. -See the [Outputs](#outputs) section below for details. +`helm` modules also export values that are available in template strings. See the [Outputs](#outputs) section below for details. ## Configuration keys @@ -789,6 +788,8 @@ valueFiles: [] ## Outputs +### Module outputs + The following keys are available via the `${modules.}` template string key for `helm` modules. @@ -851,3 +852,4 @@ The Helm release name of the service. | Type | Required | | -------- | -------- | | `string` | Yes | + diff --git a/docs/reference/module-types/kubernetes.md b/docs/reference/module-types/kubernetes.md index c8f960b589..5c49f4e40e 100644 --- a/docs/reference/module-types/kubernetes.md +++ b/docs/reference/module-types/kubernetes.md @@ -16,8 +16,7 @@ guide](../../using-garden/configuration-files.md). The [first section](#configuration-keys) lists and describes the available schema keys. The [second section](#complete-yaml-schema) contains the complete YAML schema. -`kubernetes` modules also export values that are available in template strings under `${modules..outputs}`. -See the [Outputs](#outputs) section below for details. +`kubernetes` modules also export values that are available in template strings. See the [Outputs](#outputs) section below for details. ## Configuration keys @@ -298,6 +297,8 @@ files: [] ## Outputs +### Module outputs + The following keys are available via the `${modules.}` template string key for `kubernetes` modules. @@ -350,3 +351,4 @@ The outputs defined by the module. | Type | Required | | -------- | -------- | | `object` | Yes | + diff --git a/docs/reference/module-types/maven-container.md b/docs/reference/module-types/maven-container.md index c4d7574ac0..6903fc275a 100644 --- a/docs/reference/module-types/maven-container.md +++ b/docs/reference/module-types/maven-container.md @@ -18,8 +18,7 @@ guide](../../using-garden/configuration-files.md). The [first section](#configuration-keys) lists and describes the available schema keys. The [second section](#complete-yaml-schema) contains the complete YAML schema. -`maven-container` modules also export values that are available in template strings under `${modules..outputs}`. -See the [Outputs](#outputs) section below for details. +`maven-container` modules also export values that are available in template strings. See the [Outputs](#outputs) section below for details. ## Configuration keys @@ -1082,6 +1081,8 @@ mvnOpts: [] ## Outputs +### Module outputs + The following keys are available via the `${modules.}` template string key for `maven-container` modules. @@ -1170,3 +1171,4 @@ outputs: ... deployment-image-name: "my-deployment-registry.io/my-org/my-module" ``` + diff --git a/docs/reference/module-types/openfaas.md b/docs/reference/module-types/openfaas.md index 3b54029723..29f46d9613 100644 --- a/docs/reference/module-types/openfaas.md +++ b/docs/reference/module-types/openfaas.md @@ -8,8 +8,7 @@ guide](../../using-garden/configuration-files.md). The [first section](#configuration-keys) lists and describes the available schema keys. The [second section](#complete-yaml-schema) contains the complete YAML schema. -`openfaas` modules also export values that are available in template strings under `${modules..outputs}`. -See the [Outputs](#outputs) section below for details. +`openfaas` modules also export values that are available in template strings. See the [Outputs](#outputs) section below for details. ## Configuration keys @@ -330,6 +329,8 @@ tests: ## Outputs +### Module outputs + The following keys are available via the `${modules.}` template string key for `openfaas` modules. @@ -392,3 +393,4 @@ The full URL to query this service _from within_ the cluster. | Type | Required | | -------- | -------- | | `string` | Yes | + diff --git a/docs/reference/providers/kubernetes.md b/docs/reference/providers/kubernetes.md index b4d07555e8..544ff24d75 100644 --- a/docs/reference/providers/kubernetes.md +++ b/docs/reference/providers/kubernetes.md @@ -935,8 +935,7 @@ providers: ## Outputs -The following keys are available via the `${providers.}` template string key for `kubernetes` -providers. +The following keys are available via the `${providers.}` template string key for `kubernetes` providers. ### `providers..app-namespace` diff --git a/docs/reference/providers/local-kubernetes.md b/docs/reference/providers/local-kubernetes.md index ef69ca7181..290185077a 100644 --- a/docs/reference/providers/local-kubernetes.md +++ b/docs/reference/providers/local-kubernetes.md @@ -839,8 +839,7 @@ providers: ## Outputs -The following keys are available via the `${providers.}` template string key for `local-kubernetes` -providers. +The following keys are available via the `${providers.}` template string key for `local-kubernetes` providers. ### `providers..app-namespace` diff --git a/docs/reference/template-strings.md b/docs/reference/template-strings.md index 3020b4bb5a..8ce9cd4e7b 100644 --- a/docs/reference/template-strings.md +++ b/docs/reference/template-strings.md @@ -200,6 +200,34 @@ providers: {} # modules: {} +# Runtime outputs and information from services and tasks (only resolved at runtime when deploying +# services and running tasks). +# +# Type: object +# +runtime: + # Runtime information from the services that the service/task being run depends on. + # + # Type: object + # + # Example: + # my-service: + # outputs: + # some-key: some value + # + services: {} + + # Runtime information from the tasks that the service/task being run depends on. + # + # Type: object + # + # Example: + # my-task: + # outputs: + # some-key: some value + # + tasks: {} + # A map of all variables defined in the project configuration. # # Type: object diff --git a/docs/using-garden/configuration-files.md b/docs/using-garden/configuration-files.md index 0189e1532e..19e4b32a4a 100644 --- a/docs/using-garden/configuration-files.md +++ b/docs/using-garden/configuration-files.md @@ -457,6 +457,36 @@ services: For a full reference of the keys available in template strings, please look at the [Template Strings Reference](../reference/template-strings.md). +#### Runtime outputs + +Template keys prefixed with `runtime.` have some special semantics. They are used to expose runtime outputs from services and tasks, and therefore are resolved later than other template strings. _This means that you cannot use them for some fields, such as most identifiers, because those need to be resolved before validating the configuration._ + +That caveat aside, they can be very handy when passing information between services and tasks. For example, you can pass log outputs from one task to another: + +```yaml +kind: Module +type: exec +name: module-a +tasks: + - name: prep-task + command: [echo, "output from my preparation task"] +--- +kind: Module +type: container +name: my-container +services: + - name: my-service + dependencies: [task-a] + env: + PREP_TASK_OUTPUT: ${runtime.tasks.prep-task.outputs.log} +``` + +Here the output from `prep-task` is copied to an environment variable for `my-service`. _Note that you currently need to explicitly declare `task-a` as a dependency for this to work._ + +For a practical use case, you might for example make a task that provisions some infrastructure or prepares some data, and then passes information about it to services. + +Different module types expose different output keys for their services and tasks. Please refer to the [module type reference docs](https://docs.garden.io/reference/module-types) for details. + #### Conditionals You can use conditional expressions in template strings, using the `||` operator. For example: diff --git a/garden-service/src/actions.ts b/garden-service/src/actions.ts index 0f82fe4d09..0056fa75ca 100644 --- a/garden-service/src/actions.ts +++ b/garden-service/src/actions.ts @@ -9,13 +9,13 @@ import Bluebird = require("bluebird") import chalk from "chalk" -import { fromPairs, keyBy, mapValues, omit, pickBy } from "lodash" +import { fromPairs, mapValues, omit, pickBy } from "lodash" import { PublishModuleParams, PublishResult } from "./types/plugin/module/publishModule" import { SetSecretParams, SetSecretResult } from "./types/plugin/provider/setSecret" import { validate, joi } from "./config/common" import { defaultProvider } from "./config/provider" -import { ParameterError, PluginError } from "./exceptions" +import { ParameterError, PluginError, ConfigurationError } from "./exceptions" import { ActionHandlerMap, Garden, ModuleActionHandlerMap, ModuleActionMap, PluginActionMap } from "./garden" import { LogEntry } from "./logger/log-entry" import { ProcessResults, processServices } from "./process" @@ -71,12 +71,16 @@ import { HotReloadServiceParams, HotReloadServiceResult } from "./types/plugin/s import { RunServiceParams } from "./types/plugin/service/runService" import { GetTaskResultParams } from "./types/plugin/task/getTaskResult" import { RunTaskParams, RunTaskResult } from "./types/plugin/task/runTask" -import { Service, ServiceStatus, ServiceStatusMap, getServiceRuntimeContext } from "./types/service" +import { ServiceStatus, ServiceStatusMap } from "./types/service" import { Omit } from "./util/util" import { DebugInfoMap } from "./types/plugin/provider/getDebugInfo" import { PrepareEnvironmentParams, PrepareEnvironmentResult } from "./types/plugin/provider/prepareEnvironment" import { GetPortForwardParams } from "./types/plugin/service/getPortForward" import { StopPortForwardParams } from "./types/plugin/service/stopPortForward" +import { emptyRuntimeContext, RuntimeContext } from "./runtime-context" +import { GetServiceStatusTask } from "./tasks/get-service-status" +import { getServiceStatuses } from "./tasks/base" +import { getRuntimeTemplateReferences } from "./template-string" type TypeGuard = { readonly [P in keyof (PluginActionParams | ModuleActionParams)]: (...args: any[]) => Promise @@ -208,7 +212,7 @@ export class ActionHelper implements TypeGuard { moduleType, defaultHandler: async ({ }) => ({ docs: "", - outputsSchema: joi.object().options({ allowUnknown: true }), + moduleOutputsSchema: joi.object().options({ allowUnknown: true }), schema: joi.object().options({ allowUnknown: true }), }), }) @@ -280,7 +284,8 @@ export class ActionHelper implements TypeGuard { status: "active", }) - const status = await this.getServiceStatus({ ...params, hotReload: false }) + const runtimeContext = emptyRuntimeContext + const status = await this.getServiceStatus({ ...params, runtimeContext, hotReload: false }) if (status.state === "missing") { log.setSuccess({ @@ -350,6 +355,7 @@ export class ActionHelper implements TypeGuard { const envStatus = await this.garden.getEnvironmentStatus() const serviceStatuses = await this.getServiceStatuses({ log, serviceNames }) + return { providers: envStatus, services: serviceStatuses, @@ -360,20 +366,18 @@ export class ActionHelper implements TypeGuard { { log, serviceNames }: { log: LogEntry, serviceNames?: string[] }, ): Promise { const graph = await this.garden.getConfigGraph() - const services = keyBy(await graph.getServices(serviceNames), "name") - - return Bluebird.props(mapValues(services, async (service: Service) => { - const runtimeContext = await getServiceRuntimeContext(this.garden, graph, service) - - // TODO: Some handlers expect builds to have been staged when resolving services statuses. We should - // tackle that better by getting statuses in the task graph. - await this.garden.buildDir.syncFromSrc(service.module, log) - await this.garden.buildDir.syncDependencyProducts(service.module, log) + const services = await graph.getServices(serviceNames) - // TODO: The status will be reported as "outdated" if the service was deployed with hot-reloading enabled. - // Once hot-reloading is a toggle, as opposed to an API/CLI flag, we can resolve that issue. - return this.getServiceStatus({ log, service, runtimeContext, hotReload: false }) + const tasks = services.map(service => new GetServiceStatusTask({ + force: false, + garden: this.garden, + graph, + log, + service, })) + const results = await this.garden.processTasks(tasks) + + return getServiceStatuses(results) } async deployServices( @@ -412,8 +416,7 @@ export class ActionHelper implements TypeGuard { const serviceStatuses: { [key: string]: ServiceStatus } = {} await Bluebird.map(services, async (service) => { - const runtimeContext = await getServiceRuntimeContext(this.garden, graph, service) - serviceStatuses[service.name] = await this.deleteService({ log: servicesLog, service, runtimeContext }) + serviceStatuses[service.name] = await this.deleteService({ log: servicesLog, service }) }) servicesLog.setSuccess() @@ -506,8 +509,8 @@ export class ActionHelper implements TypeGuard { { params, actionType, defaultHandler }: { params: ServiceActionHelperParams, actionType: T, defaultHandler?: ServiceActions[T] }, ): Promise { - const { log, service, runtimeContext } = params - const module = service.module + let { log, service, runtimeContext } = params + let module = omit(service.module, ["_ConfigType"]) log.verbose(`Getting ${actionType} handler for service ${service.name}`) @@ -518,9 +521,28 @@ export class ActionHelper implements TypeGuard { defaultHandler, }) + // Resolve ${runtime.*} template strings if needed. + if (runtimeContext && (await getRuntimeTemplateReferences(module)).length > 0) { + log.verbose(`Resolving runtime template strings for service '${service.name}'`) + const configContext = await this.garden.getModuleConfigContext(runtimeContext) + const graph = await this.garden.getConfigGraph({ configContext }) + service = await graph.getService(service.name) + module = service.module + + // Make sure everything has been resolved in the task config + const remainingRefs = await getRuntimeTemplateReferences(service.config) + if (remainingRefs.length > 0) { + const unresolvedStrings = remainingRefs.map(ref => `\${${ref.join(".")}}`).join(", ") + throw new ConfigurationError( + `Unable to resolve one or more runtime template values for service '${service.name}': ${unresolvedStrings}`, + { service, unresolvedStrings }, + ) + } + } + const handlerParams = { ...await this.commonParams(handler, log), - ...params, + ...params, module, runtimeContext, } @@ -537,9 +559,9 @@ export class ActionHelper implements TypeGuard { defaultHandler?: TaskActions[T], }, ): Promise { - - const { task, log } = params - const module = task.module + let { task, log } = params + const runtimeContext = params["runtimeContext"] as (RuntimeContext | undefined) + let module = omit(task.module, ["_ConfigType"]) log.verbose(`Getting ${actionType} handler for task ${module.name}.${task.name}`) @@ -550,9 +572,28 @@ export class ActionHelper implements TypeGuard { defaultHandler, }) + // Resolve ${runtime.*} template strings if needed. + if (runtimeContext && (await getRuntimeTemplateReferences(module)).length > 0) { + log.verbose(`Resolving runtime template strings for task '${task.name}'`) + const configContext = await this.garden.getModuleConfigContext(runtimeContext) + const graph = await this.garden.getConfigGraph({ configContext }) + task = await graph.getTask(task.name) + module = task.module + + // Make sure everything has been resolved in the task config + const remainingRefs = await getRuntimeTemplateReferences(task.config) + if (remainingRefs.length > 0) { + const unresolvedStrings = remainingRefs.map(ref => `\${${ref.join(".")}}`).join(", ") + throw new ConfigurationError( + `Unable to resolve one or more runtime template values for task '${task.name}': ${unresolvedStrings}`, + { task, unresolvedStrings }, + ) + } + } + const handlerParams: any = { ...await this.commonParams(handler, (params).log), - ...params, + ...params, module, task, } diff --git a/garden-service/src/commands/call.ts b/garden-service/src/commands/call.ts index 187669e6f9..8a9c9fd3ea 100644 --- a/garden-service/src/commands/call.ts +++ b/garden-service/src/commands/call.ts @@ -19,9 +19,10 @@ import { import { splitFirst } from "../util/util" import { ParameterError, RuntimeError } from "../exceptions" import { find, includes, pick } from "lodash" -import { ServiceIngress, getIngressUrl, getServiceRuntimeContext } from "../types/service" +import { ServiceIngress, getIngressUrl } from "../types/service" import dedent = require("dedent") import { printHeader } from "../logger/util" +import { emptyRuntimeContext } from "../runtime-context" const callArgs = { serviceAndPath: new StringParameter({ @@ -58,7 +59,8 @@ export class CallCommand extends Command { // TODO: better error when service doesn't exist const graph = await garden.getConfigGraph() const service = await graph.getService(serviceName) - const runtimeContext = await getServiceRuntimeContext(garden, graph, service) + // No need for full context, since we're just checking if the service is running. + const runtimeContext = emptyRuntimeContext const actions = await garden.getActionHelper() const status = await actions.getServiceStatus({ service, log, hotReload: false, runtimeContext }) diff --git a/garden-service/src/commands/delete.ts b/garden-service/src/commands/delete.ts index fa3254bba2..b9e191af74 100644 --- a/garden-service/src/commands/delete.ts +++ b/garden-service/src/commands/delete.ts @@ -16,7 +16,7 @@ import { } from "./base" import { NotFoundError } from "../exceptions" import dedent = require("dedent") -import { ServiceStatus, getServiceRuntimeContext, ServiceStatusMap } from "../types/service" +import { ServiceStatus, ServiceStatusMap } from "../types/service" import { printHeader } from "../logger/util" import { DeleteSecretResult } from "../types/plugin/provider/deleteSecret" import { EnvironmentStatusMap } from "../types/plugin/provider/getEnvironmentStatus" @@ -148,8 +148,7 @@ export class DeleteServiceCommand extends Command { const actions = await garden.getActionHelper() await Bluebird.map(services, async service => { - const runtimeContext = await getServiceRuntimeContext(garden, graph, service) - result[service.name] = await actions.deleteService({ log, service, runtimeContext }) + result[service.name] = await actions.deleteService({ log, service }) }) return { result } diff --git a/garden-service/src/commands/exec.ts b/garden-service/src/commands/exec.ts index c112f8985a..f0fda86256 100644 --- a/garden-service/src/commands/exec.ts +++ b/garden-service/src/commands/exec.ts @@ -19,7 +19,6 @@ import { StringsParameter, } from "./base" import dedent = require("dedent") -import { getServiceRuntimeContext } from "../types/service" const runArgs = { service: new StringParameter({ @@ -78,14 +77,12 @@ export class ExecCommand extends Command { const graph = await garden.getConfigGraph() const service = await graph.getService(serviceName) - const runtimeContext = await getServiceRuntimeContext(garden, graph, service) const actions = await garden.getActionHelper() const result = await actions.execInService({ log, service, command, interactive: opts.interactive, - runtimeContext, }) return { result } diff --git a/garden-service/src/commands/get/get-task-result.ts b/garden-service/src/commands/get/get-task-result.ts index 5118352106..32ac0715a9 100644 --- a/garden-service/src/commands/get/get-task-result.ts +++ b/garden-service/src/commands/get/get-task-result.ts @@ -69,7 +69,7 @@ export class GetTaskResultCommand extends Command { name: taskResult.taskName, module: taskResult.moduleName, version: taskResult.version, - output: taskResult.output, + output: taskResult.output || null, startedAt: taskResult.startedAt, completedAt: taskResult.completedAt, } diff --git a/garden-service/src/commands/get/get-test-result.ts b/garden-service/src/commands/get/get-test-result.ts index 65c6facf0d..b747ea13f4 100644 --- a/garden-service/src/commands/get/get-test-result.ts +++ b/garden-service/src/commands/get/get-test-result.ts @@ -93,7 +93,7 @@ export class GetTestResultCommand extends Command { startedAt: testResult.startedAt, completedAt: testResult.completedAt, version: testResult.version, - output: testResult.output, + output: testResult.output || null, } log.info({ data: testResult }) diff --git a/garden-service/src/commands/logs.ts b/garden-service/src/commands/logs.ts index 5aeb75bf16..54ab7819a9 100644 --- a/garden-service/src/commands/logs.ts +++ b/garden-service/src/commands/logs.ts @@ -17,11 +17,12 @@ import { import chalk from "chalk" import { ServiceLogEntry } from "../types/plugin/service/getServiceLogs" import Bluebird = require("bluebird") -import { Service, getServiceRuntimeContext } from "../types/service" +import { Service } from "../types/service" import Stream from "ts-stream" import { LoggerType } from "../logger/logger" import dedent = require("dedent") import { LogLevel } from "../logger/log-node" +import { emptyRuntimeContext } from "../runtime-context" const logsArgs = { services: new StringsParameter({ @@ -93,14 +94,19 @@ export class LogsCommand extends Command { }) const actions = await garden.getActionHelper() + const voidLog = log.placeholder(LogLevel.silly, true) await Bluebird.map(services, async (service: Service) => { - const voidLog = log.placeholder(LogLevel.silly, true) - const runtimeContext = await getServiceRuntimeContext(garden, graph, service) - const status = await actions.getServiceStatus({ log: voidLog, service, hotReload: false, runtimeContext }) + const status = await actions.getServiceStatus({ + hotReload: false, + log: voidLog, + // This shouldn't matter for this context, we're just checking if the service is up or not + runtimeContext: emptyRuntimeContext, + service, + }) if (status.state === "ready" || status.state === "outdated") { - await actions.getServiceLogs({ log, service, stream, follow, tail, runtimeContext }) + await actions.getServiceLogs({ log, service, stream, follow, tail }) } else { await stream.write({ serviceName: service.name, diff --git a/garden-service/src/commands/run/module.ts b/garden-service/src/commands/run/module.ts index b4d9f75820..4bbd7d7c81 100644 --- a/garden-service/src/commands/run/module.ts +++ b/garden-service/src/commands/run/module.ts @@ -16,10 +16,11 @@ import { CommandResult, StringsParameter, } from "../base" -import { printRuntimeContext, runtimeContextForServiceDeps } from "./run" +import { printRuntimeContext } from "./run" import { printHeader } from "../../logger/util" import { BuildTask } from "../../tasks/build" import { dedent, deline } from "../../util/string" +import { prepareRuntimeContext } from "../../runtime-context" const runArgs = { module: new StringParameter({ @@ -92,7 +93,16 @@ export class RunModuleCommand extends Command { const buildTask = new BuildTask({ garden, log, module, force: opts["force-build"] }) await garden.processTasks([buildTask]) - const runtimeContext = await runtimeContextForServiceDeps(garden, graph, module) + const dependencies = await graph.getDependencies("build", module.name, false) + + const runtimeContext = await prepareRuntimeContext({ + garden, + graph, + dependencies, + module, + serviceStatuses: {}, + taskResults: {}, + }) printRuntimeContext(log, runtimeContext) diff --git a/garden-service/src/commands/run/run.ts b/garden-service/src/commands/run/run.ts index 75082db21b..0c371d905e 100644 --- a/garden-service/src/commands/run/run.ts +++ b/garden-service/src/commands/run/run.ts @@ -7,7 +7,7 @@ */ import { safeDump } from "js-yaml" -import { RuntimeContext, prepareRuntimeContext } from "../../types/service" +import { RuntimeContext } from "../../runtime-context" import { highlightYaml } from "../../util/util" import { Command } from "../base" import { RunModuleCommand } from "./module" @@ -15,9 +15,6 @@ import { RunServiceCommand } from "./service" import { RunTaskCommand } from "./task" import { RunTestCommand } from "./test" import { LogEntry } from "../../logger/log-entry" -import { ConfigGraph } from "../../config-graph" -import { Module } from "../../types/module" -import { Garden } from "../../garden" export class RunCommand extends Command { name = "run" @@ -33,13 +30,6 @@ export class RunCommand extends Command { async action() { return {} } } -export async function runtimeContextForServiceDeps(garden: Garden, graph: ConfigGraph, module: Module) { - const depNames = module.serviceDependencyNames - const allServices = await graph.getServices() - const deps = allServices.filter(s => depNames.includes(s.name)) - return prepareRuntimeContext(garden, graph, module, deps) -} - export function printRuntimeContext(log: LogEntry, runtimeContext: RuntimeContext) { log.verbose("-----------------------------------\n") log.verbose("Environment variables:") diff --git a/garden-service/src/commands/run/service.ts b/garden-service/src/commands/run/service.ts index 0a6df847ef..ca01e44808 100644 --- a/garden-service/src/commands/run/service.ts +++ b/garden-service/src/commands/run/service.ts @@ -15,10 +15,12 @@ import { CommandResult, StringParameter, } from "../base" -import { printRuntimeContext, runtimeContextForServiceDeps } from "./run" +import { printRuntimeContext } from "./run" import dedent = require("dedent") import { printHeader } from "../../logger/util" -import { BuildTask } from "../../tasks/build" +import { DeployTask } from "../../tasks/deploy" +import { getServiceStatuses, getRunTaskResults } from "../../tasks/base" +import { prepareRuntimeContext } from "../../runtime-context" const runArgs = { service: new StringParameter({ @@ -66,10 +68,30 @@ export class RunServiceCommand extends Command { const actions = await garden.getActionHelper() - const buildTask = new BuildTask({ garden, log, module, force: opts["force-build"] }) - await garden.processTasks([buildTask]) + // Make sure all dependencies are ready and collect their outputs for the runtime context + const deployTask = new DeployTask({ + force: true, + forceBuild: opts["force-build"], + garden, + graph, + log, + service, + }) + const dependencyResults = await garden.processTasks(await deployTask.getDependencies()) + + const dependencies = await graph.getDependencies("service", serviceName, false) + const serviceStatuses = getServiceStatuses(dependencyResults) + const taskResults = getRunTaskResults(dependencyResults) + + const runtimeContext = await prepareRuntimeContext({ + garden, + graph, + dependencies, + module, + serviceStatuses, + taskResults, + }) - const runtimeContext = await runtimeContextForServiceDeps(garden, graph, module) printRuntimeContext(log, runtimeContext) const result = await actions.runService({ diff --git a/garden-service/src/commands/run/test.ts b/garden-service/src/commands/run/test.ts index 5a2377944f..2be2de26e5 100644 --- a/garden-service/src/commands/run/test.ts +++ b/garden-service/src/commands/run/test.ts @@ -22,10 +22,10 @@ import { } from "../base" import { printRuntimeContext } from "./run" import dedent = require("dedent") -import { prepareRuntimeContext } from "../../types/service" +import { prepareRuntimeContext } from "../../runtime-context" import { printHeader } from "../../logger/util" -import { BuildTask } from "../../tasks/build" -import { getTestVersion } from "../../tasks/test" +import { getTestVersion, TestTask } from "../../tasks/test" +import { getRunTaskResults, getServiceStatuses } from "../../tasks/base" const runArgs = { module: new StringParameter({ @@ -91,17 +91,37 @@ export class RunTestCommand extends Command { ) const actions = await garden.getActionHelper() - - const buildTask = new BuildTask({ garden, log, module, force: opts["force-build"] }) - await garden.processTasks([buildTask]) + const version = await getTestVersion(garden, graph, module, testConfig) + + // Make sure all dependencies are ready and collect their outputs for the runtime context + const testTask = new TestTask({ + force: true, + forceBuild: opts["force-build"], + garden, + graph, + log, + module, + testConfig, + version, + }) + const dependencyResults = await garden.processTasks(await testTask.getDependencies()) const interactive = opts.interactive - const deps = await graph.getDependencies("test", testConfig.name, false) - const runtimeContext = await prepareRuntimeContext(garden, graph, module, deps.service) + const dependencies = await graph.getDependencies("test", testConfig.name, false) - printRuntimeContext(log, runtimeContext) + const serviceStatuses = getServiceStatuses(dependencyResults) + const taskResults = getRunTaskResults(dependencyResults) - const testVersion = await getTestVersion(garden, graph, module, testConfig) + const runtimeContext = await prepareRuntimeContext({ + garden, + graph, + dependencies, + module, + serviceStatuses, + taskResults, + }) + + printRuntimeContext(log, runtimeContext) const result = await actions.testModule({ log, @@ -110,7 +130,7 @@ export class RunTestCommand extends Command { runtimeContext, silent: false, testConfig, - testVersion, + testVersion: version, }) return { result } diff --git a/garden-service/src/config/config-context.ts b/garden-service/src/config/config-context.ts index 114301d8a2..64ab0bbb66 100644 --- a/garden-service/src/config/config-context.ts +++ b/garden-service/src/config/config-context.ts @@ -17,6 +17,8 @@ import { resolveTemplateString } from "../template-string" import { Garden } from "../garden" import { ModuleVersion } from "../vcs/vcs" import { joi } from "../config/common" +import { KeyedSet } from "../util/keyed-set" +import { RuntimeContext } from "../runtime-context" export type ContextKey = string[] @@ -132,7 +134,7 @@ export abstract class ConfigContext { if (opts.allowUndefined) { return } else { - throw new ConfigurationError(`Could not find key: ${path}`, { + throw new ConfigurationError(`Could not find key: ${fullPath}`, { nodePath, fullPath, opts, @@ -157,6 +159,21 @@ export abstract class ConfigContext { } } +export class ScanContext extends ConfigContext { + foundKeys: KeyedSet + + constructor() { + super() + this.foundKeys = new KeyedSet(v => v.join(".")) + } + + async resolve({ key, nodePath }: ContextResolveParams) { + const fullKey = nodePath.concat(key) + this.foundKeys.add(fullKey) + return "${" + fullKey.join(".") + "}" + } +} + class LocalContext extends ConfigContext { @schema( joiStringMap(joi.string()).description( @@ -351,6 +368,94 @@ const exampleModule = { version: exampleVersion, } +class ServiceRuntimeContext extends ConfigContext { + @schema( + joiIdentifierMap(joiPrimitive()) + .required() + .description( + "The runtime outputs defined by the service (see individual module type " + + "[references](https://docs.garden.io/reference/module-types) for details).", + ) + .example({ "some-key": "some value" }), + ) + public outputs: PrimitiveMap + + constructor(root: ConfigContext, outputs: PrimitiveMap) { + super(root) + this.outputs = outputs + } + + async resolve(params: ContextResolveParams) { + // We're customizing the resolver so that we can ignore missing service/task outputs, but fail when an output + // on a resolved service/task doesn't exist. + const opts = { ...params.opts || {}, allowUndefined: false } + return super.resolve({ ...params, opts }) + } +} + +class TaskRuntimeContext extends ServiceRuntimeContext { + @schema( + joiIdentifierMap(joiPrimitive()) + .required() + .description( + "The runtime outputs defined by the task (see individual module type " + + "[references](https://docs.garden.io/reference/module-types) for details).", + ) + .example({ "some-key": "some value" }), + ) + public outputs: PrimitiveMap +} + +class RuntimeConfigContext extends ConfigContext { + @schema( + joiIdentifierMap(ServiceRuntimeContext.getSchema()) + .required() + .description("Runtime information from the services that the service/task being run depends on.") + .example({ "my-service": { outputs: { "some-key": "some value" } } }), + ) + public services: Map + + @schema( + joiIdentifierMap(TaskRuntimeContext.getSchema()) + .required() + .description("Runtime information from the tasks that the service/task being run depends on.") + .example({ "my-task": { outputs: { "some-key": "some value" } } }), + ) + public tasks: Map + + constructor(root: ConfigContext, runtimeContext?: RuntimeContext) { + super(root) + + this.services = new Map() + this.tasks = new Map() + + const dependencies = runtimeContext ? runtimeContext.dependencies : [] + + for (const dep of dependencies) { + if (dep.type === "service") { + this.services.set(dep.name, new ServiceRuntimeContext(this, dep.outputs)) + } else if (dep.type === "task") { + this.tasks.set(dep.name, new TaskRuntimeContext(this, dep.outputs)) + } + } + } + + async resolve(params: ContextResolveParams) { + // We're customizing the resolver so that we can ignore missing services/tasks and return the template string back + // for later resolution, but fail when an output on a resolved service/task doesn't exist. + const opts = { ...params.opts || {}, allowUndefined: true } + const res = await super.resolve({ ...params, opts }) + + if (res === undefined) { + const { key, nodePath } = params + const fullKey = nodePath.concat(key) + return "${" + fullKey.join(".") + "}" + } else { + return res + } + } +} + /** * This context is available for template strings under the `module` key in configuration files. * It is a superset of the context available under the `project` key. @@ -363,6 +468,15 @@ export class ModuleConfigContext extends ProviderConfigContext { ) public modules: Map Promise> + @schema( + RuntimeConfigContext.getSchema() + .description( + "Runtime outputs and information from services and tasks " + + "(only resolved at runtime when deploying services and running tasks).", + ), + ) + public runtime: RuntimeConfigContext + @schema( joiIdentifierMap(joiPrimitive()) .description("A map of all variables defined in the project configuration.") @@ -382,6 +496,9 @@ export class ModuleConfigContext extends ProviderConfigContext { resolvedProviders: Provider[], variables: PrimitiveMap, moduleConfigs: ModuleConfig[], + // We only supply this when resolving configuration in dependency order. + // Otherwise we pass `${runtime.*} template strings through for later resolution. + runtimeContext?: RuntimeContext, ) { super(environmentName, garden.projectName, resolvedProviders) @@ -403,6 +520,8 @@ export class ModuleConfigContext extends ProviderConfigContext { }], )) + this.runtime = new RuntimeConfigContext(this, runtimeContext) + this.var = this.variables = variables } } diff --git a/garden-service/src/docs/config.ts b/garden-service/src/docs/config.ts index 29ac2ecb4c..3bbf3b1542 100644 --- a/garden-service/src/docs/config.ts +++ b/garden-service/src/docs/config.ts @@ -22,6 +22,8 @@ import { indent, renderMarkdownTable } from "./util" import { ModuleContext } from "../config/config-context" import { defaultDotIgnoreFiles } from "../util/fs" import { providerConfigBaseSchema } from "../config/provider" +import { GardenPlugin } from "../types/plugin/plugin" +import { ModuleTypeDescription } from "../types/plugin/module/describeType" export const TEMPLATES_DIR = resolve(GARDEN_SERVICE_ROOT, "src", "docs", "templates") @@ -385,29 +387,60 @@ export function renderConfigReference(configSchema: Joi.ObjectSchema, titlePrefi * Generates the provider reference from the provider.hbs template. * The reference includes the rendered output from the config-partial.hbs template. */ -function renderProviderReference(schema: Joi.ObjectSchema, name: string, outputsSchema?: Joi.ObjectSchema) { +function renderProviderReference(name: string, plugin: GardenPlugin) { + const schema = populateProviderSchema(plugin.configSchema || providerConfigBaseSchema) + const moduleOutputsSchema = plugin.outputsSchema + const providerTemplatePath = resolve(TEMPLATES_DIR, "provider.hbs") const { markdownReference, yaml } = renderConfigReference(schema) - const outputsReference = outputsSchema - && renderConfigReference(outputsSchema, "providers..").markdownReference + const moduleOutputsReference = moduleOutputsSchema + && renderConfigReference(moduleOutputsSchema, "providers..").markdownReference const template = handlebars.compile(readFileSync(providerTemplatePath).toString()) - return template({ name, markdownReference, yaml, outputsReference }) + return template({ name, markdownReference, yaml, moduleOutputsReference }) } /** * Generates the module types reference from the module-type.hbs template. * The reference includes the rendered output from the config-partial.hbs template. */ -function renderModuleTypeReference( - schema: Joi.ObjectSchema, outputsSchema: Joi.ObjectSchema, name: string, docs: string, -) { +function renderModuleTypeReference(name: string, desc: ModuleTypeDescription) { + let { schema, docs } = desc + const moduleTemplatePath = resolve(TEMPLATES_DIR, "module-type.hbs") - const { markdownReference, yaml } = renderConfigReference(schema) - const outputsReference = renderConfigReference(outputsSchema, "modules..").markdownReference + const { markdownReference, yaml } = renderConfigReference(populateModuleSchema(schema)) + + const moduleOutputsReference = renderConfigReference( + ModuleContext.getSchema().keys({ + outputs: desc.moduleOutputsSchema! + .required() + .description("The outputs defined by the module."), + }), + "modules..", + ).markdownReference + + const serviceOutputsReference = renderConfigReference( + desc.serviceOutputsSchema!, + "runtime.services..outputs.", + ).markdownReference + + const taskOutputsReference = renderConfigReference( + desc.taskOutputsSchema!, + "runtime.tasks..outputs.", + ).markdownReference + const template = handlebars.compile(readFileSync(moduleTemplatePath).toString()) - return template({ name, docs, markdownReference, yaml, outputsReference }) + return template({ + name, + docs, + markdownReference, + yaml, + hasOutputs: moduleOutputsReference || serviceOutputsReference || taskOutputsReference, + moduleOutputsReference, + serviceOutputsReference, + taskOutputsReference, + }) } /** @@ -462,9 +495,7 @@ export async function writeConfigReferenceDocs(docsRoot: string) { const path = resolve(providerDir, `${name}.md`) console.log("->", path) - const schema = populateProviderSchema(plugin.configSchema || providerConfigBaseSchema) - const outputsSchema = plugin.outputsSchema - writeFileSync(path, renderProviderReference(schema, name, outputsSchema)) + writeFileSync(path, renderProviderReference(name, plugin)) } // Render module type docs @@ -473,23 +504,12 @@ export async function writeConfigReferenceDocs(docsRoot: string) { for (const { name } of moduleTypes) { const path = resolve(moduleTypeDir, `${name}.md`) const actions = await garden.getActionHelper() - const { docs, outputsSchema, schema, title } = await actions.describeType(name) - - const moduleOutputsSchema = ModuleContext.getSchema().keys({ - outputs: outputsSchema - .required() - .description("The outputs defined by the module."), - }) + const desc = await actions.describeType(name) console.log("->", path) - writeFileSync(path, renderModuleTypeReference( - populateModuleSchema(schema), - moduleOutputsSchema, - name, - docs, - )) + writeFileSync(path, renderModuleTypeReference(name, desc)) - readme.push(`* [${title || startCase(name.replace("-", " "))}](./${name}.md)`) + readme.push(`* [${desc.title || startCase(name.replace("-", " "))}](./${name}.md)`) } writeFileSync(resolve(moduleTypeDir, `README.md`), readme.join("\n")) diff --git a/garden-service/src/docs/templates/module-type.hbs b/garden-service/src/docs/templates/module-type.hbs index b6d814f672..e797bfde8a 100644 --- a/garden-service/src/docs/templates/module-type.hbs +++ b/garden-service/src/docs/templates/module-type.hbs @@ -7,9 +7,8 @@ guide](../../using-garden/configuration-files.md). The [first section](#configuration-keys) lists and describes the available schema keys. The [second section](#complete-yaml-schema) contains the complete YAML schema. -{{#if outputsReference}} -`{{{name}}}` modules also export values that are available in template strings under `${modules..outputs}`. -See the [Outputs](#outputs) section below for details. +{{#if hasOutputs}} +`{{{name}}}` modules also export values that are available in template strings. See the [Outputs](#outputs) section below for details. {{/if}} ## Configuration keys @@ -19,10 +18,26 @@ See the [Outputs](#outputs) section below for details. ```yaml {{{yaml}}} ``` -{{#if outputsReference}} +{{#if hasOutputs}} ## Outputs +{{/if}} +{{#if moduleOutputsReference}} + +### Module outputs The following keys are available via the `${modules.}` template string key for `{{{name}}}` modules. -{{{outputsReference}}}{{/if}} \ No newline at end of file +{{{moduleOutputsReference}}}{{/if}}{{#if serviceOutputsReference}} + +### Service outputs + +The following keys are available via the `${runtime.services.}` template string key for `{{{name}}}` module services. +Note that these are only resolved when deploying/running dependants of the service, so they are not usable for every field. +{{{serviceOutputsReference}}}{{/if}}{{#if taskOutputsReference}} + +### Task outputs + +The following keys are available via the `${runtime.tasks.}` template string key for `{{{name}}}` module tasks. +Note that these are only resolved when deploying/running dependants of the task, so they are not usable for every field. +{{{taskOutputsReference}}}{{/if}} diff --git a/garden-service/src/docs/templates/provider.hbs b/garden-service/src/docs/templates/provider.hbs index 16de037750..068d3c0af5 100644 --- a/garden-service/src/docs/templates/provider.hbs +++ b/garden-service/src/docs/templates/provider.hbs @@ -14,10 +14,9 @@ The values in the schema below are the default values. ```yaml {{{yaml}}} ``` -{{#if outputsReference}} +{{#if moduleOutputsReference}} ## Outputs -The following keys are available via the `${providers.}` template string key for `{{{name}}}` -providers. -{{{outputsReference}}}{{/if}} \ No newline at end of file +The following keys are available via the `${providers.}` template string key for `{{{name}}}` providers. +{{{moduleOutputsReference}}}{{/if}} \ No newline at end of file diff --git a/garden-service/src/garden.ts b/garden-service/src/garden.ts index a745539ec4..4aebd9630b 100644 --- a/garden-service/src/garden.ts +++ b/garden-service/src/garden.ts @@ -46,6 +46,7 @@ import { ResolveProviderTask } from "./tasks/resolve-provider" import { ActionHelper } from "./actions" import { DependencyGraph, detectCycles, cyclesToString } from "./util/validate-dependencies" import chalk from "chalk" +import { RuntimeContext } from "./runtime-context" export interface ActionHandlerMap { [actionName: string]: PluginActions[T] @@ -542,6 +543,19 @@ export class Garden { ) } + async getModuleConfigContext(runtimeContext?: RuntimeContext) { + const providers = await this.resolveProviders() + + return new ModuleConfigContext( + this, + this.environmentName, + providers, + this.variables, + Object.values(this.moduleConfigs), + runtimeContext, + ) + } + /** * Returns module configs that are registered in this context, fully resolved and configured (via their respective * plugin handlers). @@ -549,19 +563,13 @@ export class Garden { */ async resolveModuleConfigs(keys?: string[], opts: ModuleConfigResolveOpts = {}): Promise { const actions = await this.getActionHelper() - const providers = await this.resolveProviders() + await this.resolveProviders() const configs = await this.getRawModuleConfigs(keys) keys ? this.log.silly(`Resolving module configs ${keys.join(", ")}`) : this.log.silly(`Resolving module configs`) if (!opts.configContext) { - opts.configContext = new ModuleConfigContext( - this, - this.environmentName, - providers, - this.variables, - Object.values(this.moduleConfigs), - ) + opts.configContext = await this.getModuleConfigContext() } return Bluebird.map(configs, async (config) => { @@ -662,8 +670,8 @@ export class Garden { * The graph instance is immutable and represents the configuration at the point of calling this method. * For long-running processes, you need to call this again when any module or configuration has been updated. */ - async getConfigGraph() { - const modules = await this.resolveModuleConfigs() + async getConfigGraph(opts: ModuleConfigResolveOpts = {}) { + const modules = await this.resolveModuleConfigs(undefined, opts) return new ConfigGraph(this, modules) } diff --git a/garden-service/src/plugins/container/container.ts b/garden-service/src/plugins/container/container.ts index 3f8e451e99..d9e198ac75 100644 --- a/garden-service/src/plugins/container/container.ts +++ b/garden-service/src/plugins/container/container.ts @@ -34,6 +34,17 @@ export const containerModuleOutputsSchema = joi.object() .example("my-deployment-registry.io/my-org/my-module"), }) +const taskOutputsSchema = joi.object() + .keys({ + log: joi.string() + .allow("") + .default("") + .description( + "The full log from the executed task. " + + "(Pro-tip: Make it machine readable so it can be parsed by dependant tasks and services!)", + ), + }) + export async function configureContainerModule({ ctx, moduleConfig }: ConfigureModuleParams) { // validate hot reload configuration // TODO: validate this when validating this action's output @@ -171,7 +182,8 @@ async function describeType() { other module types like [helm](https://docs.garden.io/reference/module-types/helm) or [kubernetes](https://github.com/garden-io/garden/blob/master/docs/reference/module-types/kubernetes.md). `, - outputsSchema: containerModuleOutputsSchema, + moduleOutputsSchema: containerModuleOutputsSchema, schema: containerModuleSpecSchema, + taskOutputsSchema, } } diff --git a/garden-service/src/plugins/exec.ts b/garden-service/src/plugins/exec.ts index 27ad29ac61..5739febaa3 100644 --- a/garden-service/src/plugins/exec.ts +++ b/garden-service/src/plugins/exec.ts @@ -182,7 +182,7 @@ export async function testExecModule({ module, testConfig }: TestModuleParams const command = task.spec.command const startedAt = new Date() - let completedAt - let output + let completedAt: Date + let log: string if (command && command.length) { const commandResult = await execa.shell( @@ -205,10 +205,10 @@ export async function runExecTask(params: RunTaskParams): Promise ) completedAt = new Date() - output = commandResult.stdout + commandResult.stderr + log = (commandResult.stdout + commandResult.stderr).trim() } else { completedAt = startedAt - output = "" + log = "" } return { @@ -217,7 +217,10 @@ export async function runExecTask(params: RunTaskParams): Promise command, version: module.version.versionString, success: true, - output, + log, + outputs: { + log, + }, startedAt, completedAt, } @@ -229,8 +232,18 @@ async function describeType() { A simple module for executing commands in your shell. This can be a useful escape hatch if no other module type fits your needs, and you just need to execute something (as opposed to deploy it, track its status etc.). `, - outputsSchema: joi.object().keys({}), + moduleOutputsSchema: joi.object().keys({}), schema: execModuleSpecSchema, + taskOutputsSchema: joi.object() + .keys({ + log: joi.string() + .allow("") + .default("") + .description( + "The full log from the executed task. " + + "(Pro-tip: Make it machine readable so it can be parsed by dependant tasks and services!)", + ), + }), } } diff --git a/garden-service/src/plugins/kubernetes/container/build.ts b/garden-service/src/plugins/kubernetes/container/build.ts index 3e32c4c1ef..340162415b 100644 --- a/garden-service/src/plugins/kubernetes/container/build.ts +++ b/garden-service/src/plugins/kubernetes/container/build.ts @@ -202,7 +202,7 @@ const remoteBuild: BuildHandler = async (params) => { // Execute the build const buildRes = await runKaniko(provider, log, module, args) - buildLog = buildRes.output + buildLog = buildRes.log } log.silly(buildLog) diff --git a/garden-service/src/plugins/kubernetes/container/deployment.ts b/garden-service/src/plugins/kubernetes/container/deployment.ts index e8295471de..54bfc0c0f4 100644 --- a/garden-service/src/plugins/kubernetes/container/deployment.ts +++ b/garden-service/src/plugins/kubernetes/container/deployment.ts @@ -8,7 +8,7 @@ import { V1Container } from "@kubernetes/client-node" import { extend, keyBy, set } from "lodash" -import { RuntimeContext, Service, ServiceStatus } from "../../../types/service" +import { Service, ServiceStatus } from "../../../types/service" import { ContainerModule, ContainerService } from "../../container/config" import { createIngressResources } from "./ingress" import { createServiceResources } from "./service" @@ -29,6 +29,7 @@ import { DeleteServiceParams } from "../../../types/plugin/service/deleteService import { millicpuToString, kilobytesToString, prepareEnvVars } from "../util" import { gardenAnnotationKey } from "../../../util/string" import chalk from "chalk" +import { RuntimeContext } from "../../../runtime-context" export const DEFAULT_CPU_REQUEST = "10m" export const DEFAULT_MEMORY_REQUEST = "64Mi" @@ -423,7 +424,7 @@ export async function deleteService(params: DeleteServiceParams): Promise) { - const { ctx, log, service, runtimeContext } = params + const { ctx, log, service } = params const k8sCtx = ctx const context = k8sCtx.provider.config.context const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider) @@ -22,7 +23,8 @@ export async function getServiceLogs(params: GetServiceLogsParams const { ctx, log, service, command, interactive } = params const k8sCtx = ctx const provider = k8sCtx.provider - const status = await getContainerServiceStatus({ ...params, hotReload: false }) + const status = await getContainerServiceStatus({ + ...params, + // The runtime context doesn't matter here. We're just checking if the service is running. + runtimeContext: { + envVars: {}, + dependencies: [], + }, + hotReload: false, + }) const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider) // TODO: this check should probably live outside of the plugin @@ -178,7 +186,13 @@ export async function runContainerTask( timeout: task.spec.timeout || 9999, }) - const result = { ...res, taskName: task.name } + const result = { + ...res, + taskName: task.name, + outputs: { + log: res.output || "", + }, + } await storeTaskResult({ ctx, diff --git a/garden-service/src/plugins/kubernetes/container/status.ts b/garden-service/src/plugins/kubernetes/container/status.ts index b7f9aac276..a0797529a8 100644 --- a/garden-service/src/plugins/kubernetes/container/status.ts +++ b/garden-service/src/plugins/kubernetes/container/status.ts @@ -8,7 +8,7 @@ import { PluginContext } from "../../../plugin-context" import { LogEntry } from "../../../logger/log-entry" -import { RuntimeContext, Service, ServiceStatus, ForwardablePort } from "../../../types/service" +import { Service, ServiceStatus, ForwardablePort } from "../../../types/service" import { createContainerObjects } from "./deployment" import { KUBECTL_DEFAULT_TIMEOUT } from "../kubectl" import { DeploymentError } from "../../../exceptions" @@ -20,6 +20,7 @@ import { compareDeployedObjects } from "../status/status" import { getIngresses } from "./ingress" import { getAppNamespace } from "../namespace" import { KubernetesPluginContext } from "../config" +import { RuntimeContext } from "../../../runtime-context" export async function getContainerServiceStatus( { ctx, module, service, runtimeContext, log, hotReload }: GetServiceStatusParams, diff --git a/garden-service/src/plugins/kubernetes/helm/common.ts b/garden-service/src/plugins/kubernetes/helm/common.ts index 3c91550911..8c9fd6d8c0 100644 --- a/garden-service/src/plugins/kubernetes/helm/common.ts +++ b/garden-service/src/plugins/kubernetes/helm/common.ts @@ -347,9 +347,11 @@ function loadTemplate(template: string) { }) } -export function trimRunOutput(result: RunResult): RunResult { +export function trimRunOutput(result: T): T { + const log = tailString(result.log, MAX_RUN_RESULT_OUTPUT_LENGTH, true) + return { ...result, - output: tailString(result.output, MAX_RUN_RESULT_OUTPUT_LENGTH, true), + log, } } diff --git a/garden-service/src/plugins/kubernetes/helm/deployment.ts b/garden-service/src/plugins/kubernetes/helm/deployment.ts index 4240cf4014..858c8b3ae1 100644 --- a/garden-service/src/plugins/kubernetes/helm/deployment.ts +++ b/garden-service/src/plugins/kubernetes/helm/deployment.ts @@ -19,7 +19,7 @@ import { getServiceResourceSpec, getValueFileArgs, } from "./common" -import { getReleaseStatus, getServiceStatus } from "./status" +import { getReleaseStatus } from "./status" import { configureHotReload, HotReloadableResource } from "../hot-reload" import { apply } from "../kubectl" import { KubernetesPluginContext } from "../config" @@ -120,5 +120,5 @@ export async function deleteService(params: DeleteServiceParams): Promise, + { ctx, log, service, module }: HotReloadServiceParams, ): Promise { const hotReloadConfig = module.spec.hotReload @@ -170,7 +169,6 @@ export async function hotReloadContainer( ) } - await waitForContainerService(ctx, log, runtimeContext, service, true) await syncToService(ctx, service, hotReloadConfig, "Deployment", service.name, log) return {} diff --git a/garden-service/src/plugins/kubernetes/kubernetes-module/config.ts b/garden-service/src/plugins/kubernetes/kubernetes-module/config.ts index 13478d7d1e..48fd20c7f4 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes-module/config.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes-module/config.ts @@ -77,7 +77,7 @@ export async function describeType() { If you need more advanced templating features you can use the [helm](https://docs.garden.io/reference/module-types/helm) module type. `, - outputsSchema: joi.object().keys({}), + moduleOutputsSchema: joi.object().keys({}), schema: kubernetesModuleSpecSchema, } } diff --git a/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts b/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts index 56d18b4325..10229e70bb 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts @@ -120,7 +120,7 @@ async function deleteService(params: DeleteServiceParams): Promise) { diff --git a/garden-service/src/plugins/kubernetes/run.ts b/garden-service/src/plugins/kubernetes/run.ts index f4895af520..18200374af 100644 --- a/garden-service/src/plugins/kubernetes/run.ts +++ b/garden-service/src/plugins/kubernetes/run.ts @@ -107,7 +107,7 @@ export async function runPod( version: module.version.versionString, startedAt, completedAt: new Date(), - output: res.output, + log: res.output, success: res.code === 0, } } diff --git a/garden-service/src/plugins/kubernetes/system.ts b/garden-service/src/plugins/kubernetes/system.ts index d254ba1d96..99e9c5dcc8 100644 --- a/garden-service/src/plugins/kubernetes/system.ts +++ b/garden-service/src/plugins/kubernetes/system.ts @@ -69,7 +69,13 @@ export async function getSystemGarden( variables, }, commandInfo: ctx.command, - log: log.info({ section: "garden-system", msg: "Initializing...", status: "active", indent: 1 }), + log: log.debug({ + section: "garden-system", + msg: "Initializing...", + status: "active", + indent: 1, + childEntriesInheritLevel: true, + }), }) } diff --git a/garden-service/src/plugins/kubernetes/task-results.ts b/garden-service/src/plugins/kubernetes/task-results.ts index f930b47ca6..536bd77003 100644 --- a/garden-service/src/plugins/kubernetes/task-results.ts +++ b/garden-service/src/plugins/kubernetes/task-results.ts @@ -36,6 +36,18 @@ export async function getTaskResult( const result: any = deserializeValues(res.data!) // Backwards compatibility for modified result schema + if (result.output) { + result.log = result.output + } + + if (!result.outputs) { + result.outputs = {} + } + + if (!result.outputs.stdout) { + result.outputs.log = result.log + } + if (result.version.versionString) { result.version = result.version.versionString } @@ -77,6 +89,15 @@ export async function storeTaskResult( const api = await KubeApi.factory(log, provider.config.context) const namespace = await getMetadataNamespace(ctx, log, provider) + result = trimRunOutput(result) + + const data: RunTaskResult = { + ...result, + outputs: { + log: result.log, + }, + } + await upsertConfigMap({ api, namespace, @@ -87,6 +108,6 @@ export async function storeTaskResult( [gardenAnnotationKey("moduleVersion")]: module.version.versionString, [gardenAnnotationKey("version")]: taskVersion.versionString, }, - data: trimRunOutput(result), + data, }) } diff --git a/garden-service/src/plugins/local/local-docker-swarm.ts b/garden-service/src/plugins/local/local-docker-swarm.ts index 3925dad2cb..3a968686ed 100644 --- a/garden-service/src/plugins/local/local-docker-swarm.ts +++ b/garden-service/src/plugins/local/local-docker-swarm.ts @@ -174,18 +174,22 @@ export const gardenPlugin = (): GardenPlugin => ({ }, async execInService( - { ctx, service, command, runtimeContext, log }: ExecInServiceParams, + { ctx, service, command, log }: ExecInServiceParams, ) { const status = await getServiceStatus({ ctx, service, module: service.module, - runtimeContext, + // The runtime context doesn't matter here, we're just checking if the service is running. + runtimeContext: { + envVars: {}, + dependencies: [], + }, log, hotReload: false, }) - if (!status.state || status.state !== "ready") { + if (!status.state || (status.state !== "ready" && status.state !== "outdated")) { throw new DeploymentError(`Service ${service.name} is not running`, { name: service.name, state: status.state, diff --git a/garden-service/src/plugins/maven-container/maven-container.ts b/garden-service/src/plugins/maven-container/maven-container.ts index 965daefd57..418a772e10 100644 --- a/garden-service/src/plugins/maven-container/maven-container.ts +++ b/garden-service/src/plugins/maven-container/maven-container.ts @@ -113,7 +113,7 @@ async function describeType() { To use it, make sure to add the \`maven-container\` provider to your project configuration. The provider will automatically fetch and cache Maven and the appropriate OpenJDK version ahead of building. `, - outputsSchema: containerModuleOutputsSchema, + moduleOutputsSchema: containerModuleOutputsSchema, schema: mavenContainerModuleSpecSchema, } } diff --git a/garden-service/src/plugins/openfaas/config.ts b/garden-service/src/plugins/openfaas/config.ts index 0ded0835f9..70767e85ee 100644 --- a/garden-service/src/plugins/openfaas/config.ts +++ b/garden-service/src/plugins/openfaas/config.ts @@ -93,7 +93,7 @@ export async function describeType() { Deploy [OpenFaaS](https://www.openfaas.com/) functions using Garden. Requires either the \`openfaas\` or \`local-openfaas\` provider to be configured. `, - outputsSchema: openfaasModuleOutputsSchema, + moduleOutputsSchema: openfaasModuleOutputsSchema, schema: openfaasModuleSpecSchema, } } diff --git a/garden-service/src/plugins/openfaas/openfaas.ts b/garden-service/src/plugins/openfaas/openfaas.ts index f20e3addf2..6a24791a68 100644 --- a/garden-service/src/plugins/openfaas/openfaas.ts +++ b/garden-service/src/plugins/openfaas/openfaas.ts @@ -228,7 +228,7 @@ async function deployService(params: DeployServiceParams): Promi } async function deleteService(params: DeleteServiceParams): Promise { - const { ctx, log, service, runtimeContext } = params + const { ctx, log, service } = params let status let found = true @@ -237,7 +237,10 @@ async function deleteService(params: DeleteServiceParams): Promi ctx, log, service, - runtimeContext, + runtimeContext: { + envVars: {}, + dependencies: [], + }, module: service.module, hotReload: false, }) diff --git a/garden-service/src/runtime-context.ts b/garden-service/src/runtime-context.ts new file mode 100644 index 0000000000..4800ab5f7a --- /dev/null +++ b/garden-service/src/runtime-context.ts @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { getEnvVarName, uniqByName } from "./util/util" +import { + PrimitiveMap, + joiEnvVars, + joiPrimitive, + joi, + joiIdentifier, +} from "./config/common" +import { Module } from "./types/module" +import { moduleVersionSchema } from "./vcs/vcs" +import { Garden } from "./garden" +import { ConfigGraph, DependencyRelations } from "./config-graph" +import { ServiceStatus } from "./types/service" +import { RunTaskResult } from "./types/plugin/task/runTask" +import { joiArray } from "./config/common" + +interface RuntimeDependency { + moduleName: string + name: string + outputs: PrimitiveMap + type: "build" | "service" | "task" + version: string +} + +export type RuntimeContext = { + envVars: PrimitiveMap + dependencies: RuntimeDependency[], +} + +export const emptyRuntimeContext = { + envVars: {}, + dependencies: [], +} + +const runtimeDependencySchema = joi.object() + .keys({ + name: joiIdentifier() + .description("The name of the service or task."), + outputs: joiEnvVars() + .description("The outputs provided by the service (e.g. ingress URLs etc.)."), + type: joi.string() + .only("service", "task") + .description("The type of the dependency."), + version: moduleVersionSchema, + }) + +export const runtimeContextSchema = joi.object() + .options({ presence: "required" }) + .keys({ + envVars: joi.object().pattern(/.+/, joiPrimitive()) + .default(() => ({}), "{}") + .unknown(false) + .description( + "Key/value map of environment variables. Keys must be valid POSIX environment variable names " + + "(must be uppercase) and values must be primitives.", + ), + dependencies: joiArray(runtimeDependencySchema) + .description("List of all the services and tasks that this service/task/test depends on, and their metadata."), + }) + +interface PrepareRuntimeContextParams { + garden: Garden + graph: ConfigGraph + module: Module + dependencies: DependencyRelations + serviceStatuses: { [name: string]: ServiceStatus } + taskResults: { [name: string]: RunTaskResult } +} + +/** + * This function prepares the "runtime context" that's used to inform services and tasks about any dependency outputs + * and other runtime values. It includes environment variables, that can be directly passed by provider handlers to + * the underlying platform (e.g. container environments), as well as a more detailed list of all runtime + * and module dependencies and the outputs for each of them. + * + * This should be called just ahead of calling relevant service, task and test action handlers. + */ +export async function prepareRuntimeContext( + { garden, module, dependencies, serviceStatuses, taskResults }: PrepareRuntimeContextParams, +): Promise { + const { versionString } = module.version + const envVars = { + GARDEN_VERSION: versionString, + } + + for (const [key, value] of Object.entries(garden.variables)) { + const envVarName = `GARDEN_VARIABLES_${getEnvVarName(key)}` + envVars[envVarName] = value + } + + const result: RuntimeContext = { + envVars, + dependencies: [], + } + + const depModules = uniqByName([ + ...dependencies.build, + ...dependencies.service.map(d => d.module), + ...dependencies.task.map(d => d.module), + ]) + + for (const m of depModules) { + const moduleEnvName = getEnvVarName(m.name) + + for (const [key, value] of Object.entries(m.outputs)) { + envVars[`GARDEN_MODULE_${moduleEnvName}__OUTPUT_${getEnvVarName(key)}`] = value + } + } + + for (const m of dependencies.build) { + result.dependencies.push({ + moduleName: m.name, + name: m.name, + outputs: m.outputs, + type: "build", + version: m.version.versionString, + }) + } + + for (const service of dependencies.service) { + const envName = getEnvVarName(service.name) + + // If a service status is not available, we tolerate that here. That may impact dependant service status reports, + // but that is expected behavior. If a service becomes available or changes its outputs, the context changes. + // We leave it to providers to indicate what the impact of that difference is. + const status = serviceStatuses[service.name] || {} + const outputs = status.outputs || {} + + result.dependencies.push({ + moduleName: service.module.name, + name: service.name, + outputs, + type: "service", + version: service.module.version.versionString, + }) + + for (const [key, value] of Object.entries(outputs)) { + envVars[`GARDEN_SERVICE_${envName}__OUTPUT_${getEnvVarName(key)}`] = value + } + } + + for (const task of dependencies.task) { + const envName = getEnvVarName(task.name) + + // If a task result is not available, we tolerate that here. That may impact dependant service status reports, + // but that is expected behavior. If a task is later run for the first time or its output changes, the context + // changes. We leave it to providers to indicate what the impact of that difference is. + const taskResult = taskResults[task.name] || {} + const outputs = taskResult.outputs || {} + + result.dependencies.push({ + moduleName: task.module.name, + name: task.name, + outputs, + type: "task", + version: task.module.version.versionString, + }) + + for (const [key, value] of Object.entries(outputs)) { + envVars[`GARDEN_TASK_${envName}__OUTPUT_${getEnvVarName(key)}`] = value + } + } + + // Make the full list of dependencies and outputs available as JSON as well + result.envVars.GARDEN_DEPENDENCIES = JSON.stringify(result.dependencies) + + return result +} diff --git a/garden-service/src/task-graph.ts b/garden-service/src/task-graph.ts index 471e083e8e..bc6ace211c 100644 --- a/garden-service/src/task-graph.ts +++ b/garden-service/src/task-graph.ts @@ -12,7 +12,7 @@ import chalk from "chalk" import * as yaml from "js-yaml" import hasAnsi = require("has-ansi") import { flatten, merge, padEnd, pick } from "lodash" -import { BaseTask, TaskDefinitionError } from "./tasks/base" +import { BaseTask, TaskDefinitionError, TaskType } from "./tasks/base" import { LogEntry, LogEntryMetadata, TaskLogStatus } from "./logger/log-entry" import { toGardenError } from "./exceptions" @@ -22,7 +22,7 @@ import { AnalyticsHandler } from "./analytics/analytics" class TaskGraphError extends Error { } export interface TaskResult { - type: string + type: TaskType description: string key: string name: string diff --git a/garden-service/src/tasks/base.ts b/garden-service/src/tasks/base.ts index 7508af611c..2ed85d955b 100644 --- a/garden-service/src/tasks/base.ts +++ b/garden-service/src/tasks/base.ts @@ -11,8 +11,21 @@ import { ModuleVersion } from "../vcs/vcs" import { v1 as uuidv1 } from "uuid" import { Garden } from "../garden" import { LogEntry } from "../logger/log-entry" +import { pickBy, mapValues, mapKeys } from "lodash" +import { ServiceStatus } from "../types/service" +import { RunTaskResult } from "../types/plugin/task/runTask" +import { splitLast } from "../util/util" -export type TaskType = "build" | "deploy" | "publish" | "hot-reload" | "resolve-provider" | "task" | "test" +export type TaskType = + "build" | + "deploy" | + "get-service-status" | + "get-task-result" | + "hot-reload" | + "publish" | + "resolve-provider" | + "task" | + "test" export class TaskDefinitionError extends Error { } @@ -64,3 +77,21 @@ export abstract class BaseTask { abstract async process(dependencyResults: TaskResults): Promise } + +export function getServiceStatuses(dependencyResults: TaskResults): { [name: string]: ServiceStatus } { + const getServiceStatusResults = pickBy(dependencyResults, r => r.type === "get-service-status") + const deployResults = pickBy(dependencyResults, r => r.type === "deploy") + // DeployTask results take precedence over GetServiceStatusTask results, because status changes after deployment + const combined = { ...getServiceStatusResults, ...deployResults } + const statuses = mapValues(combined, r => r.output as ServiceStatus) + return mapKeys(statuses, (_, key) => splitLast(key, ".")[1]) +} + +export function getRunTaskResults(dependencyResults: TaskResults): { [name: string]: RunTaskResult } { + const storedResults = pickBy(dependencyResults, r => r.type === "get-task-result") + const runResults = pickBy(dependencyResults, r => r.type === "task") + // TaskTask results take precedence over GetTaskResultTask results + const combined = { ...storedResults, ...runResults } + const results = mapValues(combined, r => r.output as RunTaskResult) + return mapKeys(results, (_, key) => splitLast(key, ".")[1]) +} diff --git a/garden-service/src/tasks/deploy.ts b/garden-service/src/tasks/deploy.ts index 22d4766437..5090578884 100644 --- a/garden-service/src/tasks/deploy.ts +++ b/garden-service/src/tasks/deploy.ts @@ -10,13 +10,17 @@ import * as Bluebird from "bluebird" import chalk from "chalk" import { includes } from "lodash" import { LogEntry } from "../logger/log-entry" -import { BaseTask, TaskType } from "./base" -import { Service, ServiceStatus, getServiceRuntimeContext, getIngressUrl } from "../types/service" +import { BaseTask, TaskType, getServiceStatuses, getRunTaskResults } from "./base" +import { Service, ServiceStatus, getIngressUrl } from "../types/service" import { Garden } from "../garden" -import { TaskTask } from "./task" +import { TaskTask, getTaskVersion } from "./task" import { BuildTask } from "./build" import { ConfigGraph } from "../config-graph" import { startPortProxies } from "../proxy" +import { TaskResults } from "../task-graph" +import { prepareRuntimeContext } from "../runtime-context" +import { GetServiceStatusTask } from "./get-service-status" +import { GetTaskResultTask } from "./get-task-result" export interface DeployTaskParams { garden: Garden @@ -56,7 +60,7 @@ export class DeployTask extends BaseTask { const deps = await dg.getDependencies("service", this.getName(), false, (depNode) => !(depNode.type === "service" && includes(this.hotReloadServiceNames, depNode.name))) - const deployTasks = await Bluebird.map(deps.service, async (service) => { + const tasks: BaseTask[] = deps.service.map(service => { return new DeployTask({ garden: this.garden, graph: this.graph, @@ -69,8 +73,29 @@ export class DeployTask extends BaseTask { }) }) + tasks.push(new GetServiceStatusTask({ + garden: this.garden, + graph: this.graph, + log: this.log, + service: this.service, + force: false, + hotReloadServiceNames: this.hotReloadServiceNames, + })) + if (this.fromWatch && includes(this.hotReloadServiceNames, this.service.name)) { - return deployTasks + // Only need to get existing statuses and results when hot-reloading + const taskResultTasks = await Bluebird.map(deps.task, async (task) => { + return new GetTaskResultTask({ + garden: this.garden, + log: this.log, + task, + force: false, + version: await getTaskVersion(this.garden, this.graph, task), + }) + }) + + return [...tasks, ...taskResultTasks] + } else { const taskTasks = await Bluebird.map(deps.task, (task) => { return TaskTask.factory({ @@ -92,7 +117,7 @@ export class DeployTask extends BaseTask { hotReloadServiceNames: this.hotReloadServiceNames, }) - return [...deployTasks, ...taskTasks, buildTask] + return [...tasks, ...taskTasks, buildTask] } } @@ -101,32 +126,40 @@ export class DeployTask extends BaseTask { } getDescription() { - return `deploying service ${this.service.name} (from module ${this.service.module.name})` + return `deploying service '${this.service.name}' (from module '${this.service.module.name}')` } - async process(): Promise { - const log = this.log.info({ - section: this.service.name, - msg: "Checking status...", - status: "active", - }) - - // TODO: get version from build task results + async process(dependencyResults: TaskResults): Promise { let version = this.version const hotReload = includes(this.hotReloadServiceNames, this.service.name) - const runtimeContext = await getServiceRuntimeContext(this.garden, this.graph, this.service) - const actions = await this.garden.getActionHelper() + const dependencies = await this.graph.getDependencies("service", this.getName(), false) - let status = await actions.getServiceStatus({ - service: this.service, - log, - hotReload, - runtimeContext, + const serviceStatuses = getServiceStatuses(dependencyResults) + const taskResults = getRunTaskResults(dependencyResults) + + // TODO: attach runtimeContext to GetServiceTask output + const runtimeContext = await prepareRuntimeContext({ + garden: this.garden, + graph: this.graph, + dependencies, + module: this.service.module, + serviceStatuses, + taskResults, }) + const actions = await this.garden.getActionHelper() + + let status = serviceStatuses[this.service.name] + const { versionString } = version + const log = this.log.info({ + status: "active", + section: this.service.name, + msg: `Deploying version ${versionString}...`, + }) + if ( !this.force && versionString === status.version && @@ -138,8 +171,6 @@ export class DeployTask extends BaseTask { append: true, }) } else { - log.setState(`Deploying version ${versionString}...`) - try { status = await actions.deployService({ service: this.service, diff --git a/garden-service/src/tasks/get-service-status.ts b/garden-service/src/tasks/get-service-status.ts new file mode 100644 index 0000000000..8a5c69b538 --- /dev/null +++ b/garden-service/src/tasks/get-service-status.ts @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { includes } from "lodash" +import { LogEntry } from "../logger/log-entry" +import { BaseTask, TaskType, getServiceStatuses, getRunTaskResults } from "./base" +import { Service, ServiceStatus } from "../types/service" +import { Garden } from "../garden" +import { ConfigGraph } from "../config-graph" +import { TaskResults } from "../task-graph" +import { prepareRuntimeContext } from "../runtime-context" +import { GetTaskResultTask } from "./get-task-result" +import { getTaskVersion } from "./task" +import * as Bluebird from "bluebird" + +export interface GetServiceStatusTaskParams { + garden: Garden + graph: ConfigGraph + service: Service + force: boolean + log: LogEntry + hotReloadServiceNames?: string[] +} + +export class GetServiceStatusTask extends BaseTask { + type: TaskType = "get-service-status" + + private graph: ConfigGraph + private service: Service + private hotReloadServiceNames: string[] + + constructor( + { garden, graph, log, service, force, hotReloadServiceNames = [] }: GetServiceStatusTaskParams, + ) { + super({ garden, log, force, version: service.module.version }) + this.graph = graph + this.service = service + this.hotReloadServiceNames = hotReloadServiceNames + } + + async getDependencies() { + const deps = await this.graph.getDependencies("service", this.getName(), false) + + const statusTasks = deps.service.map(service => { + return new GetServiceStatusTask({ + garden: this.garden, + graph: this.graph, + log: this.log, + service, + force: false, + hotReloadServiceNames: this.hotReloadServiceNames, + }) + }) + + const taskResultTasks = await Bluebird.map(deps.task, async (task) => { + return new GetTaskResultTask({ + garden: this.garden, + log: this.log, + task, + force: false, + version: await getTaskVersion(this.garden, this.graph, task), + }) + }) + + return [...statusTasks, ...taskResultTasks] + } + + getName() { + return this.service.name + } + + getDescription() { + return `getting status for service '${this.service.name}' (from module '${this.service.module.name}')` + } + + async process(dependencyResults: TaskResults): Promise { + const log = this.log.placeholder() + + const hotReload = includes(this.hotReloadServiceNames, this.service.name) + + const dependencies = await this.graph.getDependencies("service", this.getName(), false) + + const serviceStatuses = getServiceStatuses(dependencyResults) + const taskResults = getRunTaskResults(dependencyResults) + + const runtimeContext = await prepareRuntimeContext({ + garden: this.garden, + graph: this.graph, + dependencies, + module: this.service.module, + serviceStatuses, + taskResults, + }) + + const actions = await this.garden.getActionHelper() + + // Some handlers expect builds to have been staged when resolving services statuses. + await this.garden.buildDir.syncFromSrc(this.service.module, log) + await this.garden.buildDir.syncDependencyProducts(this.service.module, log) + + let status = await actions.getServiceStatus({ + service: this.service, + log, + hotReload, + runtimeContext, + }) + + return status + } +} diff --git a/garden-service/src/tasks/get-task-result.ts b/garden-service/src/tasks/get-task-result.ts new file mode 100644 index 0000000000..f875459df2 --- /dev/null +++ b/garden-service/src/tasks/get-task-result.ts @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { LogEntry } from "../logger/log-entry" +import { BaseTask, TaskType } from "./base" +import { Garden } from "../garden" +import { Task } from "../types/task" +import { RunTaskResult } from "../types/plugin/task/runTask" +import { ModuleVersion } from "../vcs/vcs" + +export interface GetTaskResultTaskParams { + force: boolean + garden: Garden + log: LogEntry + task: Task + version: ModuleVersion +} + +export class GetTaskResultTask extends BaseTask { + type: TaskType = "get-task-result" + + private task: Task + + constructor( + { force, garden, log, task, version }: GetTaskResultTaskParams, + ) { + super({ garden, log, force, version }) + this.task = task + } + + getName() { + return this.task.name + } + + getDescription() { + return `getting task result '${this.task.name}' (from module '${this.task.module.name}')` + } + + async process(): Promise { + const log = this.log.info({ + section: this.task.name, + msg: "Checking result...", + status: "active", + }) + const actions = await this.garden.getActionHelper() + + return actions.getTaskResult({ + task: this.task, + log, + taskVersion: this.version, + }) + } +} diff --git a/garden-service/src/tasks/hot-reload.ts b/garden-service/src/tasks/hot-reload.ts index 69202fbe8b..0a2bbc7c1c 100644 --- a/garden-service/src/tasks/hot-reload.ts +++ b/garden-service/src/tasks/hot-reload.ts @@ -9,29 +9,32 @@ import chalk from "chalk" import { LogEntry } from "../logger/log-entry" import { BaseTask, TaskType } from "./base" -import { Service, getServiceRuntimeContext } from "../types/service" +import { Service } from "../types/service" import { Garden } from "../garden" import { ConfigGraph } from "../config-graph" interface Params { + force: boolean garden: Garden graph: ConfigGraph - force: boolean - service: Service + hotReloadServiceNames?: string[] log: LogEntry + service: Service } export class HotReloadTask extends BaseTask { type: TaskType = "hot-reload" - private graph: ConfigGraph + // private graph: ConfigGraph + // private hotReloadServiceNames: string[] private service: Service constructor( - { garden, graph, log, service, force }: Params, + { garden, log, service, force }: Params, ) { super({ garden, log, force, version: service.module.version }) - this.graph = graph + // this.graph = graph + // this.hotReloadServiceNames = hotReloadServiceNames || [] this.service = service } @@ -43,6 +46,25 @@ export class HotReloadTask extends BaseTask { return `hot-reloading service ${this.service.name}` } + // TODO: we will need to uncomment this once the TaskGraph is processing concurrently, but this is safe to + // omit in the meantime because dev/deploy commands are guaranteed to complete deployments before running + // hot reload tasks. + + // async getDependencies() { + // // Ensure service has been deployed before attempting to hot-reload. + // // This task should be cached and return immediately in most cases. + // return [new DeployTask({ + // fromWatch: true, + // force: false, + // forceBuild: false, + // garden: this.garden, + // graph: this.graph, + // hotReloadServiceNames: this.hotReloadServiceNames, + // log: this.log, + // service: this.service, + // })] + // } + async process(): Promise<{}> { const log = this.log.info({ section: this.service.name, @@ -50,11 +72,10 @@ export class HotReloadTask extends BaseTask { status: "active", }) - const runtimeContext = await getServiceRuntimeContext(this.garden, this.graph, this.service) const actions = await this.garden.getActionHelper() try { - await actions.hotReloadService({ log, service: this.service, runtimeContext }) + await actions.hotReloadService({ log, service: this.service }) } catch (err) { log.setError() throw err diff --git a/garden-service/src/tasks/task.ts b/garden-service/src/tasks/task.ts index afe07b4f85..986fc032a4 100644 --- a/garden-service/src/tasks/task.ts +++ b/garden-service/src/tasks/task.ts @@ -8,16 +8,18 @@ import * as Bluebird from "bluebird" import chalk from "chalk" -import { BaseTask, TaskParams, TaskType } from "../tasks/base" +import { BaseTask, TaskParams, TaskType, getServiceStatuses, getRunTaskResults } from "../tasks/base" import { Garden } from "../garden" import { Task } from "../types/task" import { DeployTask } from "./deploy" import { LogEntry } from "../logger/log-entry" -import { prepareRuntimeContext } from "../types/service" +import { prepareRuntimeContext } from "../runtime-context" import { ConfigGraph } from "../config-graph" import { ModuleVersion } from "../vcs/vcs" import { BuildTask } from "./build" import { RunTaskResult } from "../types/plugin/task/runTask" +import { TaskResults } from "../task-graph" +import { GetTaskResultTask } from "./get-task-result" export interface TaskTaskParams { garden: Garden @@ -82,7 +84,15 @@ export class TaskTask extends BaseTask { // ... to be renamed soon. }) }) - return [buildTask, ...deployTasks, ...taskTasks] + const resultTask = new GetTaskResultTask({ + force: this.force, + garden: this.garden, + log: this.log, + task: this.task, + version: this.version, + }) + + return [buildTask, ...deployTasks, ...taskTasks, resultTask] } @@ -94,31 +104,41 @@ export class TaskTask extends BaseTask { // ... to be renamed soon. return `running task ${this.task.name} in module ${this.task.module.name}` } - async process() { + async process(dependencyResults: TaskResults) { const task = this.task - const module = task.module // TODO: Re-enable this logic when we've started providing task graph results to process methods. - // const cachedResult = await this.getTaskReosult() + const cachedResult = getRunTaskResults(dependencyResults)[this.task.name] - // if (cachedResult && cachedResult.success) { - // this.log.info({ - // section: task.name, - // }).setSuccess({ msg: chalk.green("Already run") }) + if (cachedResult && cachedResult.success) { + this.log.info({ + section: task.name, + }).setSuccess({ msg: chalk.green("Already run") }) - // return cachedResult - // } + return cachedResult + } const log = this.log.info({ section: task.name, - msg: "Running", + msg: "Running...", status: "active", }) - // combine all dependencies for all services in the module, to be sure we have all the context we need - const serviceDeps = (await this.graph.getDependencies("task", this.getName(), false)).service - const runtimeContext = await prepareRuntimeContext(this.garden, this.graph, module, serviceDeps) + const dependencies = await this.graph.getDependencies("task", this.getName(), false) + + const serviceStatuses = getServiceStatuses(dependencyResults) + const taskResults = getRunTaskResults(dependencyResults) + + const runtimeContext = await prepareRuntimeContext({ + garden: this.garden, + graph: this.graph, + dependencies, + module: this.task.module, + serviceStatuses, + taskResults, + }) + const actions = await this.garden.getActionHelper() let result: RunTaskResult @@ -138,21 +158,7 @@ export class TaskTask extends BaseTask { // ... to be renamed soon. log.setSuccess({ msg: chalk.green(`Done (took ${log.getDuration(1)} sec)`), append: true }) return result - } - - // private async getTaskResult(): Promise { - // if (this.force) { - // return null - // } - - // return this.garden.actions.getTaskResult({ - // log: this.log, - // task: this.task, - // taskVersion: this.version, - // }) - // } - } /** diff --git a/garden-service/src/tasks/test.ts b/garden-service/src/tasks/test.ts index 67a801c6ca..d45fe0140d 100644 --- a/garden-service/src/tasks/test.ts +++ b/garden-service/src/tasks/test.ts @@ -16,14 +16,15 @@ import { TestConfig } from "../config/test" import { ModuleVersion } from "../vcs/vcs" import { DeployTask } from "./deploy" import { TestResult } from "../types/plugin/module/getTestResult" -import { BaseTask, TaskParams, TaskType } from "../tasks/base" -import { prepareRuntimeContext } from "../types/service" +import { BaseTask, TaskParams, TaskType, getServiceStatuses, getRunTaskResults } from "../tasks/base" +import { prepareRuntimeContext } from "../runtime-context" import { Garden } from "../garden" import { LogEntry } from "../logger/log-entry" import { ConfigGraph } from "../config-graph" import { makeTestTaskName } from "./helpers" import { BuildTask } from "./build" import { TaskTask } from "./task" +import { TaskResults } from "../task-graph" class TestError extends Error { toString() { @@ -114,7 +115,7 @@ export class TestTask extends BaseTask { return `running ${this.testConfig.name} tests in module ${this.module.name}` } - async process(): Promise { + async process(dependencyResults: TaskResults): Promise { // find out if module has already been tested const testResult = await this.getTestResult() @@ -133,8 +134,19 @@ export class TestTask extends BaseTask { status: "active", }) - const dependencies = await getTestDependencies(this.graph, this.testConfig) - const runtimeContext = await prepareRuntimeContext(this.garden, this.graph, this.module, dependencies) + const dependencies = await this.graph.getDependencies("test", this.testConfig.name, false) + const serviceStatuses = getServiceStatuses(dependencyResults) + const taskResults = getRunTaskResults(dependencyResults) + + const runtimeContext = await prepareRuntimeContext({ + garden: this.garden, + graph: this.graph, + dependencies, + module: this.module, + serviceStatuses, + taskResults, + }) + const actions = await this.garden.getActionHelper() let result: TestResult @@ -208,11 +220,6 @@ export async function getTestTasks( })) } -async function getTestDependencies(graph: ConfigGraph, testConfig: TestConfig) { - const deps = await graph.getDependencies("test", testConfig.name, false) - return deps.service -} - /** * Determine the version of the test run, based on the version of the module and each of its dependencies. */ diff --git a/garden-service/src/template-string.ts b/garden-service/src/template-string.ts index 527511ef66..82ec94ee4a 100644 --- a/garden-service/src/template-string.ts +++ b/garden-service/src/template-string.ts @@ -10,8 +10,7 @@ import lodash = require("lodash") import Bluebird = require("bluebird") import { asyncDeepMap } from "./util/util" import { GardenBaseError, ConfigurationError } from "./exceptions" -import { ConfigContext, ContextResolveOpts, ContextResolveParams } from "./config/config-context" -import { KeyedSet } from "./util/keyed-set" +import { ConfigContext, ContextResolveOpts, ScanContext } from "./config/config-context" import { uniq } from "lodash" import { Primitive } from "./config/common" @@ -92,16 +91,7 @@ export async function collectTemplateReferences(obj: T): Promi return uniq(context.foundKeys.entries()).sort() } -class ScanContext extends ConfigContext { - foundKeys: KeyedSet - - constructor() { - super() - this.foundKeys = new KeyedSet(v => v.join(".")) - } - - async resolve({ key }: ContextResolveParams) { - this.foundKeys.add(key) - return key.join(".") - } +export async function getRuntimeTemplateReferences(obj: T) { + const refs = await collectTemplateReferences(obj) + return refs.filter(ref => ref[0] === "runtime") } diff --git a/garden-service/src/types/plugin/base.ts b/garden-service/src/types/plugin/base.ts index 8f2fdaa823..07f127f2b3 100644 --- a/garden-service/src/types/plugin/base.ts +++ b/garden-service/src/types/plugin/base.ts @@ -9,7 +9,8 @@ import { LogEntry } from "../../logger/log-entry" import { PluginContext, pluginContextSchema } from "../../plugin-context" import { Module, moduleSchema } from "../module" -import { RuntimeContext, Service, serviceSchema, runtimeContextSchema } from "../service" +import { RuntimeContext, runtimeContextSchema } from "../../runtime-context" +import { Service, serviceSchema } from "../service" import { Task } from "../task" import { taskSchema } from "../../config/task" import { joi } from "../../config/common" @@ -80,7 +81,8 @@ export interface RunResult { success: boolean startedAt: Date completedAt: Date - output: string + log: string + output?: string } export const runResultSchema = joi.object() @@ -101,8 +103,11 @@ export const runResultSchema = joi.object() completedAt: joi.date() .required() .description("When the module run was completed."), - output: joi.string() + log: joi.string() .required() .allow("") .description("The output log from the run."), + output: joi.string() + .allow("") + .description("[DEPRECATED - use `log` instead] The output log from the run."), }) diff --git a/garden-service/src/types/plugin/module/describeType.ts b/garden-service/src/types/plugin/module/describeType.ts index a1ff0bae7f..9a842ab10c 100644 --- a/garden-service/src/types/plugin/module/describeType.ts +++ b/garden-service/src/types/plugin/module/describeType.ts @@ -17,8 +17,10 @@ export const describeModuleTypeParamsSchema = joi.object() export interface ModuleTypeDescription { docs: string // TODO: specify the schemas using primitives (e.g. JSONSchema/OpenAPI) and not Joi objects - outputsSchema: Joi.ObjectSchema + moduleOutputsSchema?: Joi.ObjectSchema schema: Joi.ObjectSchema + serviceOutputsSchema?: Joi.ObjectSchema, + taskOutputsSchema?: Joi.ObjectSchema, title?: string } @@ -46,18 +48,38 @@ export const describeType = { .required() .description("Documentation for the module type, in markdown format."), // TODO: specify the schemas using primitives and not Joi objects - outputsSchema: joi.object() - .default(joi.object().keys({}), "{}") - .description( - "A valid Joi schema describing the keys that each module outputs, for use in template strings " + - "(e.g. \`\${modules.my-module.outputs.some-key}\`).", - ), + moduleOutputsSchema: joi.object() + .default(() => joi.object().keys({}), "{}") + .description(dedent` + A valid Joi schema describing the keys that each module outputs at config time, for use in template strings + (e.g. \`\${modules.my-module.outputs.some-key}\`). + + If no schema is provided, an error may be thrown if a module attempts to return an output. + `), schema: joi.object() .required() .description( "A valid Joi schema describing the configuration keys for the `module` " + "field in the module's `garden.yml`.", ), + serviceOutputsSchema: joi.object() + .default(() => joi.object().keys({}), "{}") + .description(dedent` + A valid Joi schema describing the keys that each service outputs at runtime, for use in template strings + and environment variables (e.g. \`\${runtime.services.my-service.outputs.some-key}\` and + \`GARDEN_SERVICES_MY_SERVICE__OUTPUT_SOME_KEY\`). + + If no schema is provided, an error may be thrown if a service attempts to return an output. + `), + taskOutputsSchema: joi.object() + .default(() => joi.object().keys({}), "{}") + .description(dedent` + A valid Joi schema describing the keys that each task outputs at runtime, for use in template strings + and environment variables (e.g. \`\${runtime.tasks.my-task.outputs.some-key}\` and + \`GARDEN_TASKS_MY_TASK__OUTPUT_SOME_KEY\`). + + If no schema is provided, an error may be thrown if a task attempts to return an output. + `), title: joi.string() .description( "Readable title for the module type. Defaults to the title-cased type name, with dashes replaced by spaces.", diff --git a/garden-service/src/types/plugin/module/getTestResult.ts b/garden-service/src/types/plugin/module/getTestResult.ts index 370a66766d..54b9e66935 100644 --- a/garden-service/src/types/plugin/module/getTestResult.ts +++ b/garden-service/src/types/plugin/module/getTestResult.ts @@ -10,7 +10,7 @@ import { dedent, deline } from "../../../util/string" import { Module } from "../../module" import { PluginModuleActionParamsBase, moduleActionParamsSchema, RunResult, runResultSchema } from "../base" import { ModuleVersion, moduleVersionSchema } from "../../../vcs/vcs" -import { joi } from "../../../config/common" +import { joi, joiPrimitive } from "../../../config/common" export interface GetTestResultParams extends PluginModuleActionParamsBase { testName: string @@ -23,6 +23,9 @@ export interface TestResult extends RunResult { export const testResultSchema = runResultSchema .keys({ + outputs: joi.object() + .pattern(/.+/, joiPrimitive()) + .description("A map of primitive values, output from the test."), testName: joi.string() .required() .description("The name of the test that was run."), diff --git a/garden-service/src/types/plugin/module/runModule.ts b/garden-service/src/types/plugin/module/runModule.ts index d7c8543566..b13409508a 100644 --- a/garden-service/src/types/plugin/module/runModule.ts +++ b/garden-service/src/types/plugin/module/runModule.ts @@ -9,7 +9,7 @@ import { dedent } from "../../../util/string" import { Module } from "../../module" import { PluginModuleActionParamsBase, moduleActionParamsSchema, runBaseParams, runResultSchema } from "../base" -import { RuntimeContext } from "../../service" +import { RuntimeContext } from "../../../runtime-context" import { joiArray, joi } from "../../../config/common" export interface RunModuleParams extends PluginModuleActionParamsBase { diff --git a/garden-service/src/types/plugin/module/testModule.ts b/garden-service/src/types/plugin/module/testModule.ts index bc8c170e71..604af63fdb 100644 --- a/garden-service/src/types/plugin/module/testModule.ts +++ b/garden-service/src/types/plugin/module/testModule.ts @@ -9,7 +9,7 @@ import { dedent } from "../../../util/string" import { Module } from "../../module" import { PluginModuleActionParamsBase } from "../base" -import { RuntimeContext } from "../../service" +import { RuntimeContext } from "../../../runtime-context" import { ModuleVersion } from "../../../vcs/vcs" import { testConfigSchema } from "../../../config/test" import { runModuleBaseSchema } from "./runModule" diff --git a/garden-service/src/types/plugin/service/deleteService.ts b/garden-service/src/types/plugin/service/deleteService.ts index ee947c83b3..8c2e0844b0 100644 --- a/garden-service/src/types/plugin/service/deleteService.ts +++ b/garden-service/src/types/plugin/service/deleteService.ts @@ -9,11 +9,10 @@ import { PluginServiceActionParamsBase, serviceActionParamsSchema } from "../base" import { dedent } from "../../../util/string" import { Module } from "../../module" -import { RuntimeContext, runtimeContextSchema, serviceStatusSchema } from "../../service" +import { serviceStatusSchema } from "../../service" export interface DeleteServiceParams extends PluginServiceActionParamsBase { - runtimeContext: RuntimeContext } export const deleteService = { @@ -22,9 +21,6 @@ export const deleteService = { Called by the \`garden delete service\` command. `, - paramsSchema: serviceActionParamsSchema - .keys({ - runtimeContext: runtimeContextSchema, - }), + paramsSchema: serviceActionParamsSchema, resultSchema: serviceStatusSchema, } diff --git a/garden-service/src/types/plugin/service/deployService.ts b/garden-service/src/types/plugin/service/deployService.ts index 302f7152f0..cdcc5cb1df 100644 --- a/garden-service/src/types/plugin/service/deployService.ts +++ b/garden-service/src/types/plugin/service/deployService.ts @@ -9,7 +9,8 @@ import { PluginServiceActionParamsBase, serviceActionParamsSchema } from "../base" import { dedent } from "../../../util/string" import { Module } from "../../module" -import { RuntimeContext, runtimeContextSchema, serviceStatusSchema } from "../../service" +import { RuntimeContext, runtimeContextSchema } from "../../../runtime-context" +import { serviceStatusSchema } from "../../service" import { joi } from "../../../config/common" export interface DeployServiceParams diff --git a/garden-service/src/types/plugin/service/execInService.ts b/garden-service/src/types/plugin/service/execInService.ts index f2f12c9767..dc92fa56a6 100644 --- a/garden-service/src/types/plugin/service/execInService.ts +++ b/garden-service/src/types/plugin/service/execInService.ts @@ -9,13 +9,11 @@ import { PluginServiceActionParamsBase, serviceActionParamsSchema } from "../base" import { dedent } from "../../../util/string" import { Module } from "../../module" -import { RuntimeContext, runtimeContextSchema } from "../../service" import { joiArray, joi } from "../../../config/common" export interface ExecInServiceParams extends PluginServiceActionParamsBase { command: string[] - runtimeContext: RuntimeContext interactive: boolean } @@ -37,7 +35,6 @@ export const execInService = { .keys({ command: joiArray(joi.string()) .description("The command to run alongside the service."), - runtimeContext: runtimeContextSchema, interactive: joi.boolean(), }), diff --git a/garden-service/src/types/plugin/service/getServiceLogs.ts b/garden-service/src/types/plugin/service/getServiceLogs.ts index 0f13a3adfd..3fcd72b446 100644 --- a/garden-service/src/types/plugin/service/getServiceLogs.ts +++ b/garden-service/src/types/plugin/service/getServiceLogs.ts @@ -10,12 +10,11 @@ import { Stream } from "ts-stream" import { PluginServiceActionParamsBase, serviceActionParamsSchema } from "../base" import { dedent } from "../../../util/string" import { Module } from "../../module" -import { RuntimeContext, runtimeContextSchema } from "../../service" +import { runtimeContextSchema } from "../../../runtime-context" import { joi } from "../../../config/common" export interface GetServiceLogsParams extends PluginServiceActionParamsBase { - runtimeContext: RuntimeContext stream: Stream follow: boolean tail: number diff --git a/garden-service/src/types/plugin/service/getServiceStatus.ts b/garden-service/src/types/plugin/service/getServiceStatus.ts index ac6a24cab4..967838b329 100644 --- a/garden-service/src/types/plugin/service/getServiceStatus.ts +++ b/garden-service/src/types/plugin/service/getServiceStatus.ts @@ -9,7 +9,8 @@ import { PluginServiceActionParamsBase, serviceActionParamsSchema } from "../base" import { dedent } from "../../../util/string" import { Module } from "../../module" -import { RuntimeContext, runtimeContextSchema, serviceStatusSchema } from "../../service" +import { serviceStatusSchema } from "../../service" +import { RuntimeContext, runtimeContextSchema } from "../../../runtime-context" import { joi } from "../../../config/common" export type hotReloadStatus = "enabled" | "disabled" diff --git a/garden-service/src/types/plugin/service/hotReloadService.ts b/garden-service/src/types/plugin/service/hotReloadService.ts index 153b1a6189..683ff5213d 100644 --- a/garden-service/src/types/plugin/service/hotReloadService.ts +++ b/garden-service/src/types/plugin/service/hotReloadService.ts @@ -9,12 +9,10 @@ import { PluginServiceActionParamsBase, serviceActionParamsSchema } from "../base" import { dedent } from "../../../util/string" import { Module } from "../../module" -import { RuntimeContext, runtimeContextSchema } from "../../service" import { joi } from "../../../config/common" export interface HotReloadServiceParams extends PluginServiceActionParamsBase { - runtimeContext: RuntimeContext } export interface HotReloadServiceResult { } @@ -23,7 +21,6 @@ export const hotReloadService = { description: dedent` Synchronize changes directly into a running service, instead of doing a full redeploy. `, - paramsSchema: serviceActionParamsSchema - .keys({ runtimeContext: runtimeContextSchema }), + paramsSchema: serviceActionParamsSchema, resultSchema: joi.object().keys({}), } diff --git a/garden-service/src/types/plugin/service/runService.ts b/garden-service/src/types/plugin/service/runService.ts index 5ec041459c..0b4a98b8e9 100644 --- a/garden-service/src/types/plugin/service/runService.ts +++ b/garden-service/src/types/plugin/service/runService.ts @@ -9,7 +9,7 @@ import { PluginServiceActionParamsBase, serviceActionParamsSchema, runBaseParams, runResultSchema } from "../base" import { dedent } from "../../../util/string" import { Module } from "../../module" -import { RuntimeContext } from "../../service" +import { RuntimeContext } from "../../../runtime-context" export interface RunServiceParams extends PluginServiceActionParamsBase { diff --git a/garden-service/src/types/plugin/task/getTaskResult.ts b/garden-service/src/types/plugin/task/getTaskResult.ts index f7a3f08ccb..cb7123aafb 100644 --- a/garden-service/src/types/plugin/task/getTaskResult.ts +++ b/garden-service/src/types/plugin/task/getTaskResult.ts @@ -10,7 +10,7 @@ import { taskActionParamsSchema, PluginTaskActionParamsBase } from "../base" import { dedent, deline } from "../../../util/string" import { Module } from "../../module" import { moduleVersionSchema, ModuleVersion } from "../../../vcs/vcs" -import { joi } from "../../../config/common" +import { joi, joiPrimitive } from "../../../config/common" export const taskVersionSchema = moduleVersionSchema .description(deline` @@ -41,10 +41,16 @@ export const taskResultSchema = joi.object() completedAt: joi.date() .required() .description("When the task run was completed."), - output: joi.string() + log: joi.string() .required() .allow("") .description("The output log from the run."), + output: joi.string() + .allow("") + .description("[DEPRECATED - use `log` instead] The output log from the run."), + outputs: joi.object() + .pattern(/.+/, joiPrimitive()) + .description("A map of primitive values, output from the task."), }) export const getTaskResult = { diff --git a/garden-service/src/types/plugin/task/runTask.ts b/garden-service/src/types/plugin/task/runTask.ts index 7b872df952..d6839f9a91 100644 --- a/garden-service/src/types/plugin/task/runTask.ts +++ b/garden-service/src/types/plugin/task/runTask.ts @@ -9,9 +9,10 @@ import { taskActionParamsSchema, PluginTaskActionParamsBase, runBaseParams, RunResult } from "../base" import { dedent } from "../../../util/string" import { Module } from "../../module" -import { RuntimeContext } from "../../service" +import { RuntimeContext } from "../../../runtime-context" import { ModuleVersion } from "../../../vcs/vcs" import { taskVersionSchema, taskResultSchema } from "./getTaskResult" +import { PrimitiveMap } from "../../../config/common" export interface RunTaskParams extends PluginTaskActionParamsBase { interactive: boolean @@ -28,7 +29,8 @@ export interface RunTaskResult extends RunResult { success: boolean startedAt: Date completedAt: Date - output: string + log: string + outputs: PrimitiveMap } export const runTask = { diff --git a/garden-service/src/types/service.ts b/garden-service/src/types/service.ts index abba8f18a4..d40530f6bc 100644 --- a/garden-service/src/types/service.ts +++ b/garden-service/src/types/service.ts @@ -6,26 +6,14 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { getEnvVarName, uniqByName } from "../util/util" -import { - PrimitiveMap, - joiEnvVars, - joiIdentifierMap, - joiPrimitive, - joiUserIdentifier, - joi, - joiIdentifier, - joiArray, -} from "../config/common" -import { Module, getModuleKey } from "./module" +import normalizeUrl from "normalize-url" +import { format } from "url" +import { joiUserIdentifier, joi, joiIdentifier, joiArray, PrimitiveMap, joiPrimitive } from "../config/common" +import { Module } from "./module" import { ServiceConfig, serviceConfigSchema } from "../config/service" import dedent = require("dedent") -import { format } from "url" -import { moduleVersionSchema } from "../vcs/vcs" -import { Garden } from "../garden" import { uniq } from "lodash" import { ConfigGraph } from "../config-graph" -import normalizeUrl = require("normalize-url") export interface Service { name: string @@ -178,6 +166,7 @@ export interface ServiceStatus { ingresses?: ServiceIngress[], lastMessage?: string lastError?: string + outputs?: PrimitiveMap runningReplicas?: number state?: ServiceState updatedAt?: string @@ -209,6 +198,9 @@ export const serviceStatusSchema = joi.object() .description("Latest status message of the service (if any)."), lastError: joi.string() .description("Latest error status message of the service (if any)."), + outputs: joi.object() + .pattern(/.+/, joiPrimitive()) + .description("A map of primitive values, output from the service."), runningReplicas: joi.number() .description("How many replicas of the service are currently running."), state: joi.string() @@ -221,84 +213,6 @@ export const serviceStatusSchema = joi.object() .description("The Garden module version of the deployed service."), }) -export type RuntimeContext = { - envVars: PrimitiveMap - dependencies: { - [name: string]: { - version: string, - outputs: PrimitiveMap, - }, - }, -} - -const runtimeDependencySchema = joi.object() - .keys({ - version: moduleVersionSchema, - outputs: joiEnvVars() - .description("The outputs provided by the service (e.g. ingress URLs etc.)."), - }) - -export const runtimeContextSchema = joi.object() - .options({ presence: "required" }) - .keys({ - envVars: joi.object().pattern(/.+/, joiPrimitive()) - .default(() => ({}), "{}") - .unknown(false) - .description( - "Key/value map of environment variables. Keys must be valid POSIX environment variable names " + - "(must be uppercase) and values must be primitives.", - ), - dependencies: joiIdentifierMap(runtimeDependencySchema) - .description("Map of all the services that this service or test depends on, and their metadata."), - }) - -export async function prepareRuntimeContext( - garden: Garden, graph: ConfigGraph, module: Module, serviceDependencies: Service[], -): Promise { - const buildDepKeys = module.build.dependencies.map(dep => getModuleKey(dep.name, dep.plugin)) - const buildDependencies: Module[] = await graph.getModules(buildDepKeys) - const { versionString } = module.version - const envVars = { - GARDEN_VERSION: versionString, - } - - for (const [key, value] of Object.entries(garden.variables)) { - const envVarName = `GARDEN_VARIABLES_${key.replace(/-/g, "_").toUpperCase()}` - envVars[envVarName] = value - } - - const output: RuntimeContext = { - envVars, - dependencies: {}, - } - - const deps = output.dependencies - const depModules = uniqByName([...buildDependencies, ...serviceDependencies.map(s => s.module)]) - - for (const m of depModules) { - deps[m.name] = { - version: m.version.versionString, - outputs: m.outputs, - } - } - - for (const [name, dep] of Object.entries(deps)) { - const moduleEnvName = getEnvVarName(name) - - for (const [key, value] of Object.entries(dep.outputs)) { - const envVarName = `GARDEN_MODULE_${moduleEnvName}__${key}`.toUpperCase() - envVars[envVarName] = value - } - } - - return output -} - -export async function getServiceRuntimeContext(garden: Garden, graph: ConfigGraph, service: Service) { - const deps = await graph.getDependencies("service", service.name, false) - return prepareRuntimeContext(garden, graph, service.module, deps.service) -} - export function getIngressUrl(ingress: ServiceIngress) { return normalizeUrl(format({ protocol: ingress.protocol, diff --git a/garden-service/src/util/util.ts b/garden-service/src/util/util.ts index 8314cf9999..f9daf3d774 100644 --- a/garden-service/src/util/util.ts +++ b/garden-service/src/util/util.ts @@ -354,7 +354,7 @@ export function uniqByName(array: T[]): T[] { * (e.g. "my-service" -> "MY_SERVICE") */ export function getEnvVarName(identifier: string) { - return identifier.replace("-", "_").toUpperCase() + return identifier.replace(/-/g, "_").toUpperCase() } /** diff --git a/garden-service/test/helpers.ts b/garden-service/test/helpers.ts index c76953259e..347a42af94 100644 --- a/garden-service/test/helpers.ts +++ b/garden-service/test/helpers.ts @@ -75,7 +75,7 @@ async function runModule(params: RunModuleParams): Promise { moduleName: params.module.name, command: [...(params.command || []), ...params.args], completedAt: testNow, - output: "OK", + log: "OK", version: params.module.version.versionString, startedAt: testNow, success: true, @@ -207,6 +207,9 @@ export const testPlugin: PluginFactory = (): GardenPlugin => { return { ...result, taskName: task.name, + outputs: { + log: result.log, + }, } }, diff --git a/garden-service/test/unit/data/test-projects/exec-task-outputs/garden.yml b/garden-service/test/unit/data/test-projects/exec-task-outputs/garden.yml new file mode 100644 index 0000000000..5cff439fd2 --- /dev/null +++ b/garden-service/test/unit/data/test-projects/exec-task-outputs/garden.yml @@ -0,0 +1,2 @@ +kind: Project +name: exec-task-outputs diff --git a/garden-service/test/unit/data/test-projects/exec-task-outputs/module-a/garden.yml b/garden-service/test/unit/data/test-projects/exec-task-outputs/module-a/garden.yml new file mode 100644 index 0000000000..02d1d8edd8 --- /dev/null +++ b/garden-service/test/unit/data/test-projects/exec-task-outputs/module-a/garden.yml @@ -0,0 +1,6 @@ +kind: Module +name: module-a +type: exec +tasks: + - name: task-a + command: [echo, task-a-output] diff --git a/garden-service/test/unit/data/test-projects/exec-task-outputs/module-b/garden.yml b/garden-service/test/unit/data/test-projects/exec-task-outputs/module-b/garden.yml new file mode 100644 index 0000000000..03f83fb9cf --- /dev/null +++ b/garden-service/test/unit/data/test-projects/exec-task-outputs/module-b/garden.yml @@ -0,0 +1,7 @@ +kind: Module +name: module-b +type: exec +tasks: + - name: task-b + dependencies: [task-a] + command: [echo, "${runtime.tasks.task-a.outputs.log}"] diff --git a/garden-service/test/unit/src/actions.ts b/garden-service/test/unit/src/actions.ts index e0159a2318..575b6bb281 100644 --- a/garden-service/test/unit/src/actions.ts +++ b/garden-service/test/unit/src/actions.ts @@ -6,7 +6,8 @@ import { moduleActionDescriptions, pluginActionDescriptions, } from "../../../src/types/plugin/plugin" -import { RuntimeContext, Service, getServiceRuntimeContext } from "../../../src/types/service" +import { Service, ServiceState } from "../../../src/types/service" +import { RuntimeContext, prepareRuntimeContext } from "../../../src/runtime-context" import { expectError, makeTestGardenA } from "../../helpers" import { ActionHelper } from "../../../src/actions" import { Garden } from "../../../src/garden" @@ -38,7 +39,19 @@ describe("ActionHelper", () => { const graph = await garden.getConfigGraph() module = await graph.getModule("module-a") service = await graph.getService("service-a") - runtimeContext = await getServiceRuntimeContext(garden, graph, service) + runtimeContext = await prepareRuntimeContext({ + garden, + graph, + dependencies: { + build: [], + service: [], + task: [], + test: [], + }, + module, + serviceStatuses: {}, + taskResults: {}, + }) task = await graph.getTask("task-a") }) @@ -126,7 +139,7 @@ describe("ActionHelper", () => { service, runtimeContext: { envVars: { FOO: "bar" }, - dependencies: {}, + dependencies: [], }, }) expect(result).to.eql({}) @@ -143,14 +156,14 @@ describe("ActionHelper", () => { interactive: true, runtimeContext: { envVars: { FOO: "bar" }, - dependencies: {}, + dependencies: [], }, }) expect(result).to.eql({ moduleName: module.name, command, completedAt: now, - output: "bla bla", + log: "bla bla", success: true, startedAt: now, version: module.version.versionString, @@ -166,7 +179,7 @@ describe("ActionHelper", () => { interactive: true, runtimeContext: { envVars: { FOO: "bar" }, - dependencies: {}, + dependencies: [], }, silent: false, testConfig: { @@ -181,7 +194,10 @@ describe("ActionHelper", () => { moduleName: module.name, command: [], completedAt: now, - output: "bla bla", + log: "bla bla", + outputs: { + log: "bla bla", + }, success: true, startedAt: now, testName: "test", @@ -202,7 +218,10 @@ describe("ActionHelper", () => { moduleName: module.name, command: [], completedAt: now, - output: "bla bla", + log: "bla bla", + outputs: { + log: "bla bla", + }, success: true, startedAt: now, testName: "test", @@ -218,6 +237,11 @@ describe("ActionHelper", () => { const result = await actions.getServiceStatus({ log, service, runtimeContext, hotReload: false }) expect(result).to.eql({ forwardablePorts: [], state: "ready" }) }) + + it("should resolve runtime template strings", async () => { + const result = await actions.getServiceStatus({ log, service, runtimeContext, hotReload: false }) + expect(result).to.eql({ forwardablePorts: [], state: "ready" }) + }) }) describe("deployService", () => { @@ -263,14 +287,14 @@ describe("ActionHelper", () => { interactive: true, runtimeContext: { envVars: { FOO: "bar" }, - dependencies: {}, + dependencies: [], }, }) expect(result).to.eql({ moduleName: service.module.name, command: ["foo"], completedAt: now, - output: "bla bla", + log: "bla bla", success: true, startedAt: now, version: service.module.version.versionString, @@ -279,27 +303,55 @@ describe("ActionHelper", () => { }) }) - describe("runTask", () => { - it("should correctly call the corresponding plugin handler", async () => { - const result = await actions.runTask({ - log, - task, - interactive: true, - runtimeContext: { - envVars: { FOO: "bar" }, - dependencies: {}, - }, - taskVersion: task.module.version, + describe("task actions", () => { + describe("getTaskResult", () => { + it("should correctly call the corresponding plugin handler", async () => { + const result = await actions.getTaskResult({ + log, + task, + taskVersion: task.module.version, + }) + expect(result).to.eql({ + moduleName: task.module.name, + taskName: task.name, + command: ["foo"], + completedAt: now, + log: "bla bla", + outputs: { + log: "bla bla", + }, + success: true, + startedAt: now, + version: task.module.version.versionString, + }) }) - expect(result).to.eql({ - moduleName: task.module.name, - taskName: task.name, - command: ["foo"], - completedAt: now, - output: "bla bla", - success: true, - startedAt: now, - version: task.module.version.versionString, + }) + + describe("runTask", () => { + it("should correctly call the corresponding plugin handler", async () => { + const result = await actions.runTask({ + log, + task, + interactive: true, + runtimeContext: { + envVars: { FOO: "bar" }, + dependencies: [], + }, + taskVersion: task.module.version, + }) + expect(result).to.eql({ + moduleName: task.module.name, + taskName: task.name, + command: ["foo"], + completedAt: now, + log: "bla bla", + outputs: { + log: "bla bla", + }, + success: true, + startedAt: now, + version: task.module.version.versionString, + }) }) }) }) @@ -363,6 +415,195 @@ describe("ActionHelper", () => { ) }) }) + + describe("callServiceHandler", () => { + it("should interpolate runtime template strings", async () => { + const emptyActions = new ActionHelper(garden, {}) + + garden["moduleConfigs"]["module-a"].spec.foo = "\${runtime.services.service-b.outputs.foo}" + + const graph = await garden.getConfigGraph() + const serviceA = await graph.getService("service-a") + const serviceB = await graph.getService("service-b") + + const _runtimeContext = await prepareRuntimeContext({ + garden, + graph, + dependencies: { + build: [], + service: [serviceB], + task: [], + test: [], + }, + module: serviceA.module, + serviceStatuses: { + "service-b": { + outputs: { foo: "bar" }, + }, + }, + taskResults: {}, + }) + + await emptyActions["callServiceHandler"]({ + actionType: "deployService", // Doesn't matter which one it is + params: { + service: serviceA, + runtimeContext: _runtimeContext, + log, + hotReload: false, + force: false, + }, + defaultHandler: async (params) => { + expect(params.module.spec.foo).to.equal("bar") + + return { forwardablePorts: [], state: "ready" } + }, + }) + }) + + it("should throw if one or more runtime variables remain unresolved after re-resolution", async () => { + const emptyActions = new ActionHelper(garden, {}) + + garden["moduleConfigs"]["module-a"].spec.services[0].foo = "\${runtime.services.service-b.outputs.foo}" + + const graph = await garden.getConfigGraph() + const serviceA = await graph.getService("service-a") + + const _runtimeContext = await prepareRuntimeContext({ + garden, + graph, + dependencies: { + build: [], + service: [], + task: [], + test: [], + }, + module: serviceA.module, + serviceStatuses: {}, + taskResults: {}, + }) + + await expectError( + () => emptyActions["callServiceHandler"]({ + actionType: "deployService", // Doesn't matter which one it is + params: { + service: serviceA, + runtimeContext: _runtimeContext, + log, + hotReload: false, + force: false, + }, + defaultHandler: async () => { + return {} as any + }, + }), + (err) => expect(err.message).to.equal( + "Unable to resolve one or more runtime template values for service 'service-a': " + + "\${runtime.services.service-b.outputs.foo}", + ), + ) + }) + }) + + describe("callTaskHandler", () => { + it("should interpolate runtime template strings", async () => { + const emptyActions = new ActionHelper(garden, {}) + + garden["moduleConfigs"]["module-a"].spec.tasks[0].foo = "\${runtime.services.service-b.outputs.foo}" + + const graph = await garden.getConfigGraph() + const taskA = await graph.getTask("task-a") + const serviceB = await graph.getService("service-b") + + const _runtimeContext = await prepareRuntimeContext({ + garden, + graph, + dependencies: { + build: [], + service: [serviceB], + task: [], + test: [], + }, + module: taskA.module, + serviceStatuses: { + "service-b": { + outputs: { foo: "bar" }, + }, + }, + taskResults: {}, + }) + + await emptyActions["callTaskHandler"]({ + actionType: "runTask", + params: { + task: taskA, + runtimeContext: _runtimeContext, + log, + taskVersion: task.module.version, + interactive: false, + }, + defaultHandler: async (params) => { + expect(params.task.spec.foo).to.equal("bar") + + return { + moduleName: "module-b", + taskName: "task-b", + command: [], + outputs: { moo: "boo" }, + success: true, + version: task.module.version.versionString, + startedAt: new Date(), + completedAt: new Date(), + log: "boo", + } + }, + }) + }) + + it("should throw if one or more runtime variables remain unresolved after re-resolution", async () => { + const emptyActions = new ActionHelper(garden, {}) + + garden["moduleConfigs"]["module-a"].spec.tasks[0].foo = "\${runtime.services.service-b.outputs.foo}" + + const graph = await garden.getConfigGraph() + const taskA = await graph.getTask("task-a") + + const _runtimeContext = await prepareRuntimeContext({ + garden, + graph, + dependencies: { + build: [], + service: [], + task: [], + test: [], + }, + module: taskA.module, + // Omitting the service-b outputs here + serviceStatuses: {}, + taskResults: {}, + }) + + await expectError( + () => emptyActions["callTaskHandler"]({ + actionType: "runTask", + params: { + task: taskA, + runtimeContext: _runtimeContext, + log, + taskVersion: task.module.version, + interactive: false, + }, + defaultHandler: async () => { + return {} as any + }, + }), + (err) => expect(err.message).to.equal( + "Unable to resolve one or more runtime template values for task 'task-a': " + + "\${runtime.services.service-b.outputs.foo}", + ), + ) + }) + }) }) const testPlugin: PluginFactory = async () => ({ @@ -406,7 +647,9 @@ const testPlugin: PluginFactory = async () => ({ validate(params, moduleActionDescriptions.describeType.paramsSchema) return { docs: "bla bla bla", - outputsSchema: joi.object(), + moduleOutputsSchema: joi.object(), + serviceOutputsSchema: joi.object(), + taskOutputsSchema: joi.object(), schema: joi.object(), title: "Bla", } @@ -461,7 +704,7 @@ const testPlugin: PluginFactory = async () => ({ moduleName: params.module.name, command: params.args, completedAt: now, - output: "bla bla", + log: "bla bla", success: true, startedAt: now, version: params.module.version.versionString, @@ -474,7 +717,10 @@ const testPlugin: PluginFactory = async () => ({ moduleName: params.module.name, command: [], completedAt: now, - output: "bla bla", + log: "bla bla", + outputs: { + log: "bla bla", + }, success: true, startedAt: now, testName: params.testConfig.name, @@ -488,7 +734,10 @@ const testPlugin: PluginFactory = async () => ({ moduleName: params.module.name, command: [], completedAt: now, - output: "bla bla", + log: "bla bla", + outputs: { + log: "bla bla", + }, success: true, startedAt: now, testName: params.testName, @@ -530,7 +779,7 @@ const testPlugin: PluginFactory = async () => ({ moduleName: params.module.name, command: ["foo"], completedAt: now, - output: "bla bla", + log: "bla bla", success: true, startedAt: now, version: params.module.version.versionString, @@ -558,7 +807,10 @@ const testPlugin: PluginFactory = async () => ({ taskName: params.task.name, command: ["foo"], completedAt: now, - output: "bla bla", + log: "bla bla", + outputs: { + log: "bla bla", + }, success: true, startedAt: now, version: params.module.version.versionString, @@ -573,7 +825,10 @@ const testPlugin: PluginFactory = async () => ({ taskName: params.task.name, command: ["foo"], completedAt: now, - output: "bla bla", + log: "bla bla", + outputs: { + log: "bla bla", + }, success: true, startedAt: now, version: params.module.version.versionString, diff --git a/garden-service/test/unit/src/commands/deploy.ts b/garden-service/test/unit/src/commands/deploy.ts index 3429d62d95..64a19e8108 100644 --- a/garden-service/test/unit/src/commands/deploy.ts +++ b/garden-service/test/unit/src/commands/deploy.ts @@ -12,7 +12,7 @@ import { RunTaskParams, RunTaskResult } from "../../../../src/types/plugin/task/ const placeholderTimestamp = new Date() -const placeholderTaskResult = (moduleName, taskName, command) => ({ +const placeholderTaskResult = (moduleName: string, taskName: string, command: string[]) => ({ moduleName, taskName, command, @@ -20,7 +20,10 @@ const placeholderTaskResult = (moduleName, taskName, command) => ({ success: true, startedAt: placeholderTimestamp, completedAt: placeholderTimestamp, - output: "out", + log: "out", + outputs: { + log: "out", + }, }) const taskResultA = placeholderTaskResult("module-a", "task-a", ["echo", "A"]) @@ -110,8 +113,34 @@ describe("DeployCommand", () => { "build.module-a": { fresh: true, buildLog: "A" }, "build.module-b": { fresh: true, buildLog: "B" }, "build.module-c": {}, + "get-task-result.task-a": null, + "get-task-result.task-c": null, "task.task-a": taskResultA, "task.task-c": taskResultC, + "get-service-status.service-a": { + forwardablePorts: [], + ingresses: [ + { + hostname: "service-a.test-project-b.local.app.garden", + path: "/path-a", + port: 80, + protocol: "http", + }, + ], + state: "ready", + }, + "get-service-status.service-b": { + forwardablePorts: [], + state: "unknown", + }, + "get-service-status.service-c": { + forwardablePorts: [], + state: "ready", + }, + "get-service-status.service-d": { + forwardablePorts: [], + state: "unknown", + }, "deploy.service-a": { forwardablePorts: [], version: "1", state: "ready" }, "deploy.service-b": { forwardablePorts: [], version: "1", state: "ready" }, "deploy.service-c": { forwardablePorts: [], version: "1", state: "ready" }, @@ -148,8 +177,26 @@ describe("DeployCommand", () => { "build.module-a": { fresh: true, buildLog: "A" }, "build.module-b": { fresh: true, buildLog: "B" }, "build.module-c": {}, + "get-task-result.task-a": null, + "get-task-result.task-c": null, "task.task-a": taskResultA, "task.task-c": taskResultC, + "get-service-status.service-a": { + forwardablePorts: [], + ingresses: [ + { + hostname: "service-a.test-project-b.local.app.garden", + path: "/path-a", + port: 80, + protocol: "http", + }, + ], + state: "ready", + }, + "get-service-status.service-b": { + forwardablePorts: [], + state: "unknown", + }, "deploy.service-a": { forwardablePorts: [], version: "1", state: "ready" }, "deploy.service-b": { forwardablePorts: [], version: "1", state: "ready" }, }) diff --git a/garden-service/test/unit/src/commands/run/module.ts b/garden-service/test/unit/src/commands/run/module.ts index c780ed6df3..fbb19ff4a7 100644 --- a/garden-service/test/unit/src/commands/run/module.ts +++ b/garden-service/test/unit/src/commands/run/module.ts @@ -36,7 +36,7 @@ describe("RunModuleCommand", () => { moduleName: "module-a", command: [], completedAt: testNow, - output: "OK", + log: "OK", version: testModuleVersion.versionString, startedAt: testNow, success: true, @@ -60,7 +60,7 @@ describe("RunModuleCommand", () => { moduleName: "module-a", command: ["my", "command"], completedAt: testNow, - output: "OK", + log: "OK", version: testModuleVersion.versionString, startedAt: testNow, success: true, @@ -84,7 +84,7 @@ describe("RunModuleCommand", () => { moduleName: "module-a", command: ["/bin/sh", "-c", "my", "command"], completedAt: testNow, - output: "OK", + log: "OK", version: testModuleVersion.versionString, startedAt: testNow, success: true, diff --git a/garden-service/test/unit/src/commands/run/service.ts b/garden-service/test/unit/src/commands/run/service.ts index a30a1f1740..4201ec71ac 100644 --- a/garden-service/test/unit/src/commands/run/service.ts +++ b/garden-service/test/unit/src/commands/run/service.ts @@ -37,7 +37,7 @@ describe("RunServiceCommand", () => { moduleName: "module-a", command: ["service-a"], completedAt: testNow, - output: "OK", + log: "OK", version: testModuleVersion.versionString, startedAt: testNow, success: true, diff --git a/garden-service/test/unit/src/commands/run/task.ts b/garden-service/test/unit/src/commands/run/task.ts index bb4394cad4..34c6b4b459 100644 --- a/garden-service/test/unit/src/commands/run/task.ts +++ b/garden-service/test/unit/src/commands/run/task.ts @@ -22,7 +22,10 @@ describe("RunTaskCommand", () => { const expected = { command: ["echo", "OK"], moduleName: "module-a", - output: "OK", + log: "OK", + outputs: { + log: "OK", + }, success: true, taskName: "task-a", } diff --git a/garden-service/test/unit/src/commands/test.ts b/garden-service/test/unit/src/commands/test.ts index 49634392de..2262545ab2 100644 --- a/garden-service/test/unit/src/commands/test.ts +++ b/garden-service/test/unit/src/commands/test.ts @@ -25,7 +25,7 @@ describe("commands.test", () => { }, "test.module-a.unit": { success: true, - output: "OK", + log: "OK", }, "build.module-b": { fresh: true, @@ -34,11 +34,11 @@ describe("commands.test", () => { "build.module-c": {}, "test.module-b.unit": { success: true, - output: "OK", + log: "OK", }, "test.module-c.unit": { success: true, - output: "OK", + log: "OK", }, })).to.be.true }) @@ -64,7 +64,7 @@ describe("commands.test", () => { }, "test.module-a.unit": { success: true, - output: "OK", + log: "OK", }, })).to.be.true }) @@ -90,22 +90,22 @@ describe("commands.test", () => { }, "test.module-a.integration": { success: true, - output: "OK", + log: "OK", }, "test.module-c.integ": { success: true, - output: "OK", + log: "OK", }, })).to.be.true expect(isSubset(taskResultOutputs(result!), { "test.module-a.unit": { success: true, - output: "OK", + log: "OK", }, "test.module-c.unit": { success: true, - output: "OK", + log: "OK", }, })).to.be.false }) diff --git a/garden-service/test/unit/src/config/config-context.ts b/garden-service/test/unit/src/config/config-context.ts index cb2063baa7..c2136180ec 100644 --- a/garden-service/test/unit/src/config/config-context.ts +++ b/garden-service/test/unit/src/config/config-context.ts @@ -11,6 +11,8 @@ import { expectError, makeTestGardenA } from "../../../helpers" import { Garden } from "../../../../src/garden" import { join } from "path" import { joi } from "../../../../src/config/common" +import { prepareRuntimeContext } from "../../../../src/runtime-context" +import { Service } from "../../../../src/types/service" type TestValue = string | ConfigContext | TestValues | TestValueFunction type TestValueFunction = () => TestValue | Promise @@ -120,6 +122,23 @@ describe("ConfigContext", () => { await expectError(() => resolveKey(c, ["nested", "bla"]), "configuration") }) + it("should show full template string in error when unable to resolve in nested context", async () => { + class Nested extends ConfigContext { } + class Context extends ConfigContext { + nested: ConfigContext + + constructor(parent?: ConfigContext) { + super(parent) + this.nested = new Nested(this) + } + } + const c = new Context() + await expectError( + () => resolveKey(c, ["nested", "bla"]), + (err) => expect(err.message).to.equal("Could not find key: nested.bla"), + ) + }) + it("should resolve template strings", async () => { const c = new TestContext({ foo: "bar", @@ -280,4 +299,100 @@ describe("ModuleConfigContext", () => { it("should should resolve a project variable under the var alias", async () => { expect(await c.resolve({ key: ["var", "some"], nodePath: [], opts: {} })).to.equal("variable") }) + + context("runtimeContext is not set", () => { + it("should return runtime template strings unchanged", async () => { + expect(await c.resolve({ key: ["runtime", "some", "key"], nodePath: [], opts: {} })) + .to.equal("\${runtime.some.key}") + }) + }) + + context("runtimeContext is set", () => { + let withRuntime: ModuleConfigContext + let serviceA: Service + + before(async () => { + const graph = await garden.getConfigGraph() + serviceA = await graph.getService("service-a") + const serviceB = await graph.getService("service-b") + const taskB = await graph.getTask("task-b") + + const runtimeContext = await prepareRuntimeContext({ + garden, + graph, + dependencies: { + build: [], + service: [serviceB], + task: [taskB], + test: [], + }, + module: serviceA.module, + serviceStatuses: { + "service-b": { + outputs: { foo: "bar" }, + }, + }, + taskResults: { + "task-b": { + moduleName: "module-b", + taskName: "task-b", + command: [], + outputs: { moo: "boo" }, + success: true, + version: taskB.module.version.versionString, + startedAt: new Date(), + completedAt: new Date(), + log: "boo", + }, + }, + }) + + withRuntime = new ModuleConfigContext( + garden, + garden.environmentName, + await garden.resolveProviders(), + garden.variables, + Object.values((garden).moduleConfigs), + runtimeContext, + ) + }) + + it("should resolve service outputs", async () => { + const result = await withRuntime.resolve({ + key: ["runtime", "services", "service-b", "outputs", "foo"], + nodePath: [], + opts: {}, + }) + expect(result).to.equal("bar") + }) + + it("should resolve task outputs", async () => { + const result = await withRuntime.resolve({ + key: ["runtime", "tasks", "task-b", "outputs", "moo"], + nodePath: [], + opts: {}, + }) + expect(result).to.equal("boo") + }) + + it("should return the template string back if a service's outputs haven't been resolved", async () => { + const result = await withRuntime.resolve({ + key: ["runtime", "services", "not-ready", "outputs", "foo"], + nodePath: [], + opts: {}, + }) + expect(result).to.equal("\${runtime.services.not-ready.outputs.foo}") + }) + + it("should throw when a service's outputs have been resolved but an output key is not found", async () => { + await expectError( + () => withRuntime.resolve({ + key: ["runtime", "services", "service-b", "outputs", "boo"], + nodePath: [], + opts: {}, + }), + (err) => expect(err.message).to.equal("Could not find key: runtime.services.service-b.outputs.boo"), + ) + }) + }) }) diff --git a/garden-service/test/unit/src/plugins/exec.ts b/garden-service/test/unit/src/plugins/exec.ts index 6d3f232cb1..4f92d62ede 100644 --- a/garden-service/test/unit/src/plugins/exec.ts +++ b/garden-service/test/unit/src/plugins/exec.ts @@ -6,14 +6,10 @@ import { GARDEN_BUILD_VERSION_FILENAME } from "../../../../src/constants" import { LogEntry } from "../../../../src/logger/log-entry" import { keyBy } from "lodash" import { ConfigGraph } from "../../../../src/config-graph" -import { - writeModuleVersionFile, - readModuleVersionFile, -} from "../../../../src/vcs/vcs" -import { - dataDir, - makeTestGarden, -} from "../../../helpers" +import { getDataDir } from "../../../helpers" +import { TaskTask } from "../../../../src/tasks/task" +import { writeModuleVersionFile, readModuleVersionFile } from "../../../../src/vcs/vcs" +import { dataDir, makeTestGarden } from "../../../helpers" describe("exec plugin", () => { const projectRoot = resolve(dataDir, "test-project-exec") @@ -136,6 +132,26 @@ describe("exec plugin", () => { ]) }) + it("should propagate task logs to runtime outputs", async () => { + const _garden = await makeTestGarden(await getDataDir("test-projects", "exec-task-outputs")) + const _graph = await _garden.getConfigGraph() + const taskB = await _graph.getTask("task-b") + + const taskTask = new TaskTask({ + garden: _garden, + graph: _graph, + task: taskB, + log: _garden.log, + force: false, + forceBuild: false, + version: taskB.module.version, + }) + const results = await _garden.processTasks([taskTask]) + + // Task A echoes "task-a-output" and Task B echoes the output from Task A + expect(results["task.task-b"].output.outputs.log).to.equal("task-a-output") + }) + describe("getBuildStatus", () => { it("should read a build version file if it exists", async () => { const module = await graph.getModule(moduleName) diff --git a/garden-service/test/unit/src/runtime-context.ts b/garden-service/test/unit/src/runtime-context.ts new file mode 100644 index 0000000000..3b2cc620e6 --- /dev/null +++ b/garden-service/test/unit/src/runtime-context.ts @@ -0,0 +1,226 @@ +import { Garden } from "../../../src/garden" +import { makeTestGardenA } from "../../helpers" +import { ConfigGraph } from "../../../src/config-graph" +import { prepareRuntimeContext } from "../../../src/runtime-context" +import { expect } from "chai" + +describe("prepareRuntimeContext", () => { + let garden: Garden + let graph: ConfigGraph + + before(async () => { + garden = await makeTestGardenA() + graph = await garden.getConfigGraph() + }) + + it("should add the module version to the output envVars", async () => { + const module = await graph.getModule("module-a") + + const runtimeContext = await prepareRuntimeContext({ + garden, + graph, + module, + dependencies: { + build: [], + service: [], + task: [], + test: [], + }, + serviceStatuses: {}, + taskResults: {}, + }) + + expect(runtimeContext.envVars.GARDEN_VERSION).to.equal(module.version.versionString) + }) + + it("should add project variables to the output envVars", async () => { + const module = await graph.getModule("module-a") + + garden["variables"]["my-var"] = "foo" + + const runtimeContext = await prepareRuntimeContext({ + garden, + graph, + module, + dependencies: { + build: [], + service: [], + task: [], + test: [], + }, + serviceStatuses: {}, + taskResults: {}, + }) + + expect(runtimeContext.envVars.GARDEN_VARIABLES_MY_VAR).to.equal("foo") + }) + + it("should add outputs for every build dependency output", async () => { + const module = await graph.getModule("module-a") + const moduleB = await graph.getModule("module-b") + + moduleB.outputs = { "my-output": "meep" } + + const runtimeContext = await prepareRuntimeContext({ + garden, + graph, + module, + dependencies: { + build: [moduleB], + service: [], + task: [], + test: [], + }, + serviceStatuses: {}, + taskResults: {}, + }) + + expect(runtimeContext.dependencies).to.eql([ + { + moduleName: "module-b", + name: "module-b", + outputs: moduleB.outputs, + type: "build", + version: moduleB.version.versionString, + }, + ]) + + expect(runtimeContext.envVars.GARDEN_MODULE_MODULE_B__OUTPUT_MY_OUTPUT).to.equal("meep") + }) + + it("should add outputs for every service dependency runtime output", async () => { + const module = await graph.getModule("module-a") + const serviceB = await graph.getService("service-b") + + const outputs = { + "my-output": "moop", + } + + const runtimeContext = await prepareRuntimeContext({ + garden, + graph, + module, + dependencies: { + build: [], + service: [serviceB], + task: [], + test: [], + }, + serviceStatuses: { + "service-b": { + outputs, + }, + }, + taskResults: {}, + }) + + expect(runtimeContext.dependencies).to.eql([ + { + moduleName: "module-b", + name: "service-b", + outputs, + type: "service", + version: serviceB.module.version.versionString, + }, + ]) + + expect(runtimeContext.envVars.GARDEN_SERVICE_SERVICE_B__OUTPUT_MY_OUTPUT).to.equal("moop") + }) + + it("should add outputs for every task dependency runtime output", async () => { + const module = await graph.getModule("module-a") + const taskB = await graph.getTask("task-b") + + const outputs = { + "my-output": "mewp", + } + + const runtimeContext = await prepareRuntimeContext({ + garden, + graph, + module, + dependencies: { + build: [], + service: [], + task: [taskB], + test: [], + }, + serviceStatuses: {}, + taskResults: { + "task-b": { + command: ["foo"], + completedAt: new Date(), + log: "mewp", + moduleName: "module-b", + outputs, + startedAt: new Date(), + success: true, + taskName: "task-b", + version: taskB.module.version.versionString, + }, + }, + }) + + expect(runtimeContext.dependencies).to.eql([ + { + moduleName: "module-b", + name: "task-b", + outputs, + type: "task", + version: taskB.module.version.versionString, + }, + ]) + + expect(runtimeContext.envVars.GARDEN_TASK_TASK_B__OUTPUT_MY_OUTPUT).to.equal("mewp") + }) + + it("should add output envVars for every module dependency, incl. task and service dependency modules", async () => { + const module = await graph.getModule("module-a") + const serviceB = await graph.getService("service-b") + const taskB = await graph.getTask("task-c") + + serviceB.module.outputs = { "module-output-b": "meep" } + taskB.module.outputs = { "module-output-c": "moop" } + + const runtimeContext = await prepareRuntimeContext({ + garden, + graph, + module, + dependencies: { + build: [], + service: [serviceB], + task: [taskB], + test: [], + }, + serviceStatuses: {}, + taskResults: {}, + }) + + expect(runtimeContext.envVars.GARDEN_MODULE_MODULE_B__OUTPUT_MODULE_OUTPUT_B).to.equal("meep") + expect(runtimeContext.envVars.GARDEN_MODULE_MODULE_C__OUTPUT_MODULE_OUTPUT_C).to.equal("moop") + }) + + it("should output the list of dependencies as an env variable", async () => { + const module = await graph.getModule("module-a") + const serviceB = await graph.getService("service-b") + const taskB = await graph.getTask("task-c") + + const runtimeContext = await prepareRuntimeContext({ + garden, + graph, + module, + dependencies: { + build: [], + service: [serviceB], + task: [taskB], + test: [], + }, + serviceStatuses: {}, + taskResults: {}, + }) + + const parsed = JSON.parse(runtimeContext.envVars.GARDEN_DEPENDENCIES as string) + + expect(parsed).to.eql(runtimeContext.dependencies) + }) +}) diff --git a/garden-service/test/unit/src/tasks/helpers.ts b/garden-service/test/unit/src/tasks/helpers.ts index c0129eab2d..6f967187b9 100644 --- a/garden-service/test/unit/src/tasks/helpers.ts +++ b/garden-service/test/unit/src/tasks/helpers.ts @@ -53,6 +53,7 @@ describe("TaskHelpers", () => { expect(await dependencyBaseKeys(tasks)).to.eql([ "build.build-dependency", "build.good-morning", + "get-service-status.good-morning", "task.good-morning-task", ].sort()) })