refactor(core): allow plugins to augment the module graph

Providers are now able to augment the stack graph using a
new `augmentGraph` handler, which is called in dependency
order for each provider that specifies it. This allows
providers to add modules (and by extension services, tasks,
tests), and to add dependency relations between existing
modules and services/tasks.

See the new augmentGraph plugin handler for details on usage.

Also made a small refactor to no longer require a build
handler for every module type, and to avoid unnecessary
build tasks when a module doesn't require building.

garden-service/src/actions.ts

@@ -97,6 +97,7 @@ import { DeleteServiceTask, deletedServiceStatuses } from "./tasks/delete-servic
 import { realpath, writeFile } from "fs-extra"
 import { relative, join } from "path"
 import { getArtifactKey } from "./util/artifacts"
+import { AugmentGraphResult, AugmentGraphParams } from "./types/plugin/provider/augmentGraph"
 
 const maxArtifactLogLines = 5 // max number of artifacts to list in console after task+test runs
 
@@ -181,7 +182,7 @@ export class ActionRouter implements TypeGuard {
 
     const handlerParams: PluginActionParams["configureProvider"] = {
       ...omit(params, ["pluginName"]),
-      base: this.wrapBase(handler.base),
+      base: this.wrapBase(handler!.base),
     }
 
     const result = (<Function>handler)(handlerParams)
@@ -191,6 +192,17 @@ export class ActionRouter implements TypeGuard {
     return result
   }
 
+  async augmentGraph(params: RequirePluginName<ActionRouterParams<AugmentGraphParams>>): Promise<AugmentGraphResult> {
+    const { pluginName } = params
+
+    return this.callActionHandler({
+      actionType: "augmentGraph",
+      pluginName,
+      params: omit(params, ["pluginName"]),
+      defaultHandler: async () => ({ addBuildDependencies: [], addRuntimeDependencies: [], addModules: [] }),
+    })
+  }
+
   async getEnvironmentStatus(
     params: RequirePluginName<ActionRouterParams<GetEnvironmentStatusParams>> & { ctx?: PluginContext }
   ): Promise<EnvironmentStatus> {
@@ -283,7 +295,11 @@ export class ActionRouter implements TypeGuard {
   }
 
   async build<T extends Module>(params: ModuleActionRouterParams<BuildModuleParams<T>>): Promise<BuildResult> {
-    return this.callModuleHandler({ params, actionType: "build" })
+    return this.callModuleHandler({
+      params,
+      actionType: "build",
+      defaultHandler: async () => ({}),
+    })
   }
 
   async publishModule<T extends Module>(
@@ -516,7 +532,7 @@ export class ActionRouter implements TypeGuard {
     log: LogEntry
     serviceNames?: string[]
   }): Promise<ServiceStatusMap> {
-    const graph = await this.garden.getConfigGraph()
+    const graph = await this.garden.getConfigGraph(log)
     const services = await graph.getServices(serviceNames)
 
     const tasks = services.map(
@@ -540,7 +556,7 @@ export class ActionRouter implements TypeGuard {
     forceBuild = false,
     log,
   }: DeployServicesParams): Promise<ProcessResults> {
-    const graph = await this.garden.getConfigGraph()
+    const graph = await this.garden.getConfigGraph(log)
     const services = await graph.getServices(serviceNames)
 
     return processServices({
@@ -566,7 +582,7 @@ export class ActionRouter implements TypeGuard {
    * Deletes all services and cleans up the specified environment.
    */
   async deleteEnvironment(log: LogEntry) {
-    const graph = await this.garden.getConfigGraph()
+    const graph = await this.garden.getConfigGraph(log)
 
     const servicesLog = log.info({ msg: chalk.white("Deleting services..."), status: "active" })
 
@@ -678,7 +694,7 @@ export class ActionRouter implements TypeGuard {
     })
 
     const handlerParams: PluginActionParams[T] = {
-      ...(await this.commonParams(handler, params.log)),
+      ...(await this.commonParams(handler!, params.log)),
       ...(<any>params),
     }
 
@@ -746,7 +762,7 @@ export class ActionRouter implements TypeGuard {
     if (runtimeContext && (await getRuntimeTemplateReferences(module)).length > 0) {
       log.silly(`Resolving runtime template strings for service '${service.name}'`)
       const configContext = await this.garden.getModuleConfigContext(runtimeContext)
-      const graph = await this.garden.getConfigGraph({ configContext })
+      const graph = await this.garden.getConfigGraph(log, { configContext })
       service = await graph.getService(service.name)
       module = service.module
 
@@ -803,7 +819,7 @@ export class ActionRouter implements TypeGuard {
     if (runtimeContext && (await getRuntimeTemplateReferences(module)).length > 0) {
       log.silly(`Resolving runtime template strings for task '${task.name}'`)
       const configContext = await this.garden.getModuleConfigContext(runtimeContext)
-      const graph = await this.garden.getConfigGraph({ configContext })
+      const graph = await this.garden.getConfigGraph(log, { configContext })
       task = await graph.getTask(task.name)
       module = task.module
 
@@ -971,15 +987,17 @@ export class ActionRouter implements TypeGuard {
   /**
    * Get the last configured handler for the specified action (and optionally module type).
    */
-  private async getActionHandler<T extends keyof WrappedPluginActionHandlers>({
+  async getActionHandler<T extends keyof WrappedPluginActionHandlers>({
     actionType,
     pluginName,
     defaultHandler,
+    throwIfMissing = true,
   }: {
     actionType: T
     pluginName: string
     defaultHandler?: PluginActionHandlers[T]
-  }): Promise<WrappedPluginActionHandlers[T]> {
+    throwIfMissing?: boolean
+  }): Promise<WrappedPluginActionHandlers[T] | null> {
     const handlers = Object.values(await this.getActionHandlers(actionType, pluginName))
 
     // Since we only allow retrieving by plugin name, the length is always either 0 or 1
@@ -995,6 +1013,10 @@ export class ActionRouter implements TypeGuard {
       )
     }
 
+    if (!throwIfMissing) {
+      return null
+    }
+
     const errorDetails = {
       requestedHandlerType: actionType,
       environment: this.garden.environmentName,
@@ -1013,9 +1035,9 @@ export class ActionRouter implements TypeGuard {
   }
 
   /**
-   * Get the last configured handler for the specified action.
+   * Get the configured handler for the specified action.
    */
-  private async getModuleActionHandler<T extends keyof ModuleAndRuntimeActionHandlers>({
+  async getModuleActionHandler<T extends keyof ModuleAndRuntimeActionHandlers>({
     actionType,
     moduleType,
     pluginName,

garden-service/src/commands/build.ts

@@ -15,12 +15,13 @@ import {
   StringsParameter,
   PrepareParams,
 } from "./base"
-import { BuildTask } from "../tasks/build"
+import { getBuildTasks } from "../tasks/build"
 import { TaskResults } from "../task-graph"
 import dedent = require("dedent")
 import { processModules } from "../process"
 import { printHeader } from "../logger/util"
 import { startServer, GardenServer } from "../server/server"
+import { flatten } from "lodash"
 
 const buildArguments = {
   modules: new StringsParameter({
@@ -81,24 +82,25 @@ export class BuildCommand extends Command<Args, Opts> {
 
     await garden.clearBuilds()
 
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const modules = await graph.getModules(args.modules)
     const moduleNames = modules.map((m) => m.name)
 
     const results = await processModules({
       garden,
-      graph: await garden.getConfigGraph(),
+      graph: await garden.getConfigGraph(log),
       log,
       footerLog,
       modules,
       watch: opts.watch,
-      handler: async (_, module) => [new BuildTask({ garden, log, module, force: opts.force })],
+      handler: async (_, module) => getBuildTasks({ garden, log, module, force: opts.force }),
       changeHandler: async (_, module) => {
         const dependantModules = (await graph.getDependants("build", module.name, true)).build
-        return [module]
+        const tasks = [module]
           .concat(dependantModules)
           .filter((m) => moduleNames.includes(m.name))
-          .map((m) => new BuildTask({ garden, log, module: m, force: true }))
+          .map((m) => getBuildTasks({ garden, log, module: m, force: true }))
+        return flatten(await Promise.all(tasks))
       },
     })
 

garden-service/src/commands/call.ts

@@ -52,7 +52,7 @@ export class CallCommand extends Command<Args> {
     let [serviceName, path] = splitFirst(args.serviceAndPath, "/")
 
     // TODO: better error when service doesn't exist
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const service = await graph.getService(serviceName)
     // No need for full context, since we're just checking if the service is running.
     const runtimeContext = emptyRuntimeContext

garden-service/src/commands/delete.ts

@@ -126,7 +126,7 @@ export class DeleteServiceCommand extends Command {
   `
 
   async action({ garden, log, headerLog, args }: CommandParams<DeleteServiceArgs>): Promise<CommandResult> {
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const services = await graph.getServices(args.services)
 
     if (services.length === 0) {

garden-service/src/commands/deploy.ts

@@ -102,7 +102,7 @@ export class DeployCommand extends Command<Args, Opts> {
       this.server.setGarden(garden)
     }
 
-    const initGraph = await garden.getConfigGraph()
+    const initGraph = await garden.getConfigGraph(log)
     const services = await initGraph.getServices(args.services)
 
     if (services.length === 0) {

garden-service/src/commands/dev.ts

@@ -105,7 +105,7 @@ export class DevCommand extends Command<Args, Opts> {
   async action({ garden, log, footerLog, opts }: CommandParams<Args, Opts>): Promise<CommandResult> {
     this.server.setGarden(garden)
 
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const modules = await graph.getModules()
 
     if (modules.length === 0) {

garden-service/src/commands/exec.ts

@@ -72,7 +72,7 @@ export class ExecCommand extends Command<Args> {
       "runner"
     )
 
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const service = await graph.getService(serviceName)
     const actions = await garden.getActionRouter()
     const result = await actions.execInService({

garden-service/src/commands/get/get-config.ts

@@ -14,7 +14,7 @@ export class GetConfigCommand extends Command {
   help = "Outputs the fully resolved configuration for this project and environment."
 
   async action({ garden, log }: CommandParams): Promise<CommandResult<ConfigDump>> {
-    const config = await garden.dumpConfig()
+    const config = await garden.dumpConfig(log)
 
     // TODO: do a nicer print of this by default
     log.info({ data: config })

garden-service/src/commands/get/get-graph.ts

@@ -19,7 +19,7 @@ export class GetGraphCommand extends Command {
   help = "Outputs the dependency relationships specified in this project's garden.yml files."
 
   async action({ garden, log }: CommandParams): Promise<CommandResult<GraphOutput>> {
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const renderedGraph = graph.render()
     const output: GraphOutput = {
       nodes: renderedGraph.nodes,

garden-service/src/commands/get/get-status.ts

@@ -50,7 +50,7 @@ export class GetStatusCommand extends Command {
     let result: AllEnvironmentStatus
 
     if (opts.output) {
-      const graph = await garden.getConfigGraph()
+      const graph = await garden.getConfigGraph(log)
       result = await Bluebird.props({
         ...status,
         tests: getTestStatuses(garden, graph, log),

garden-service/src/commands/get/get-task-result.ts

@@ -43,7 +43,7 @@ export class GetTaskResultCommand extends Command<Args> {
   }: CommandParams<Args>): Promise<CommandResult<GetTaskResultCommandResult>> {
     const taskName = args.name
 
-    const graph: ConfigGraph = await garden.getConfigGraph()
+    const graph: ConfigGraph = await garden.getConfigGraph(log)
     const task = await graph.getTask(taskName)
 
     const actions = await garden.getActionRouter()

garden-service/src/commands/get/get-tasks.ts

@@ -61,7 +61,7 @@ export class GetTasksCommand extends Command<Args> {
   }
 
   async action({ args, garden, log }: CommandParams<Args>): Promise<CommandResult> {
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const tasks = await graph.getTasks(args.tasks)
     const taskModuleNames = uniq(tasks.map((t) => t.module.name))
     const modules = sortBy(await graph.getModules(taskModuleNames), (m) => m.name)

garden-service/src/commands/get/get-test-result.ts

@@ -55,7 +55,7 @@ export class GetTestResultCommand extends Command<Args> {
       "heavy_check_mark"
     )
 
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const actions = await garden.getActionRouter()
 
     const module = await graph.getModule(moduleName)

garden-service/src/commands/link/module.ts

@@ -50,7 +50,7 @@ export class LinkModuleCommand extends Command<Args> {
     const sourceType = "module"
 
     const { module: moduleName, path } = args
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const moduleToLink = await graph.getModule(moduleName)
 
     const isRemote = [moduleToLink].filter(hasRemoteSource)[0]

garden-service/src/commands/logs.ts

@@ -63,7 +63,7 @@ export class LogsCommand extends Command<Args, Opts> {
 
   async action({ garden, log, args, opts }: CommandParams<Args, Opts>): Promise<CommandResult<ServiceLogEntry[]>> {
     const { follow, tail } = opts
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const services = await graph.getServices(args.services)
 
     const result: ServiceLogEntry[] = []

garden-service/src/commands/publish.ts

@@ -64,7 +64,7 @@ export class PublishCommand extends Command<Args, Opts> {
   }: CommandParams<Args, Opts>): Promise<CommandResult<TaskResults>> {
     printHeader(headerLog, "Publish modules", "rocket")
 
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const modules = await graph.getModules(args.modules)
 
     const results = await publishModules(garden, log, modules, !!opts["force-build"], !!opts["allow-dirty"])

garden-service/src/commands/run/module.ts

@@ -11,7 +11,7 @@ import { RunResult } from "../../types/plugin/base"
 import { BooleanParameter, Command, CommandParams, StringParameter, CommandResult, StringsParameter } from "../base"
 import { printRuntimeContext } from "./run"
 import { printHeader } from "../../logger/util"
-import { BuildTask } from "../../tasks/build"
+import { getBuildTasks } from "../../tasks/build"
 import { dedent, deline } from "../../util/string"
 import { prepareRuntimeContext } from "../../runtime-context"
 
@@ -74,7 +74,7 @@ export class RunModuleCommand extends Command<Args, Opts> {
   async action({ garden, log, headerLog, args, opts }: CommandParams<Args, Opts>): Promise<CommandResult<RunResult>> {
     const moduleName = args.module
 
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const module = await graph.getModule(moduleName)
 
     const msg = args.arguments
@@ -85,13 +85,13 @@ export class RunModuleCommand extends Command<Args, Opts> {
 
     const actions = await garden.getActionRouter()
 
-    const buildTask = new BuildTask({
+    const buildTasks = await getBuildTasks({
       garden,
       log,
       module,
       force: opts["force-build"],
     })
-    await garden.processTasks([buildTask])
+    await garden.processTasks(buildTasks)
 
     const dependencies = await graph.getDependencies("build", module.name, false)
 

garden-service/src/commands/run/service.ts

@@ -50,7 +50,7 @@ export class RunServiceCommand extends Command<Args, Opts> {
 
   async action({ garden, log, headerLog, args, opts }: CommandParams<Args, Opts>): Promise<CommandResult<RunResult>> {
     const serviceName = args.service
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const service = await graph.getService(serviceName)
     const module = service.module
 

garden-service/src/commands/run/task.ts

@@ -51,7 +51,7 @@ export class RunTaskCommand extends Command<Args, Opts> {
     args,
     opts,
   }: CommandParams<Args, Opts>): Promise<CommandResult<TaskResult | null>> {
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const task = await graph.getTask(args.task)
 
     const msg = `Running task ${chalk.white(task.name)}`

garden-service/src/commands/run/test.ts

@@ -64,7 +64,7 @@ export class RunTestCommand extends Command<Args, Opts> {
     const moduleName = args.module
     const testName = args.test
 
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
     const module = await graph.getModule(moduleName)
 
     const testConfig = findByName(module.testConfigs, testName)

garden-service/src/commands/scan.ts

@@ -17,7 +17,8 @@ export class ScanCommand extends Command {
   help = "Scans your project and outputs an overview of all modules."
 
   async action({ garden, log }: CommandParams): Promise<CommandResult<DeepPrimitiveMap>> {
-    const modules = (await garden.resolveModuleConfigs()).map((m) => {
+    const graph = await garden.getConfigGraph(log)
+    const modules = (await graph.getModules()).map((m) => {
       return omit(m, ["_ConfigType", "cacheContext", "serviceNames", "taskNames"])
     })
 

garden-service/src/commands/test.ts

@@ -101,7 +101,7 @@ export class TestCommand extends Command<Args, Opts> {
       this.server.setGarden(garden)
     }
 
-    const graph = await garden.getConfigGraph()
+    const graph = await garden.getConfigGraph(log)
 
     let modules: Module[]
     if (args.modules) {