improvement(core): make module resolution faster to reduce startup time

This makes module template string resolution happen more explicitly
in dependency order, and avoids repeated resolution of the same module
while resolving template strings. Avoids issues with large/complex
projects, such as slow startups.

Also performed the following optimizations:

* Removed all async/promise code from template string resolution.
* Made a few optimizations in the TaskGraph.
* Switched to much simpler cycle detection in the TaskGraph.

garden-service/src/actions.ts

@@ -100,6 +100,8 @@ import { AugmentGraphResult, AugmentGraphParams } from "./types/plugin/provider/
 import { DeployTask } from "./tasks/deploy"
 import { BuildDependencyConfig } from "./config/module"
 import { Profile } from "./util/profiling"
+import { ConfigGraph } from "./config-graph"
+import { ModuleConfigContext } from "./config/config-context"
 
 const maxArtifactLogLines = 5 // max number of artifacts to list in console after task+test runs
 
@@ -108,6 +110,7 @@ type TypeGuard = {
 }
 
 export interface DeployServicesParams {
+  graph: ConfigGraph
   log: LogEntry
   serviceNames?: string[]
   force?: boolean
@@ -539,7 +542,7 @@ export class ActionRouter implements TypeGuard {
     serviceNames?: string[]
   }): Promise<ServiceStatusMap> {
     const graph = await this.garden.getConfigGraph(log)
-    const services = await graph.getServices({ names: serviceNames })
+    const services = graph.getServices({ names: serviceNames })
 
     const tasks = services.map(
       (service) =>
@@ -551,14 +554,13 @@ export class ActionRouter implements TypeGuard {
           service,
         })
     )
-    const results = await this.garden.processTasks(tasks)
+    const results = await this.garden.processTasks(tasks, { throwOnError: true })
 
     return getServiceStatuses(results)
   }
 
-  async deployServices({ serviceNames, force = false, forceBuild = false, log }: DeployServicesParams) {
-    const graph = await this.garden.getConfigGraph(log)
-    const services = await graph.getServices({ names: serviceNames })
+  async deployServices({ graph, serviceNames, force = false, forceBuild = false, log }: DeployServicesParams) {
+    const services = graph.getServices({ names: serviceNames })
 
     const tasks = services.map(
       (service) =>
@@ -585,7 +587,7 @@ export class ActionRouter implements TypeGuard {
 
     const servicesLog = log.info({ msg: chalk.white("Deleting services..."), status: "active" })
 
-    const services = await graph.getServices({ names })
+    const services = graph.getServices({ names })
 
     const deleteResults = await this.garden.processTasks(
       services.map((service) => {
@@ -749,7 +751,7 @@ export class ActionRouter implements TypeGuard {
     const handlerParams = {
       ...(await this.commonParams(handler, (<any>params).log)),
       ...params,
-      module: omit(module, ["_ConfigType"]),
+      module: omit(module, ["_config"]),
     }
 
     log.silly(`Calling ${actionType} handler for module ${module.name}`)
@@ -768,7 +770,7 @@ export class ActionRouter implements TypeGuard {
     defaultHandler?: ServiceActionHandlers[T]
   }) {
     let { log, service, runtimeContext } = params
-    let module = omit(service.module, ["_ConfigType"])
+    let module = omit(service.module, ["_config"])
 
     log.silly(`Getting ${actionType} handler for service ${service.name}`)
 
@@ -786,13 +788,24 @@ export class ActionRouter implements TypeGuard {
 
     if (!runtimeContextIsEmpty && (await getRuntimeTemplateReferences(module)).length > 0) {
       log.silly(`Resolving runtime template strings for service '${service.name}'`)
-      const configContext = await this.garden.getModuleConfigContext(runtimeContext)
-      // We first allow partial resolution on the full config graph, and then resolve the service config itself
-      // below with allowPartial=false to ensure all required strings are resolved.
-      const graph = await this.garden.getConfigGraph(log, { configContext, allowPartial: true })
-      service = await graph.getService(service.name)
+
+      const providers = await this.garden.resolveProviders()
+      const graph = await this.garden.getConfigGraph(log, runtimeContext)
+      service = graph.getService(service.name)
       module = service.module
-      service.config = await resolveTemplateStrings(service.config, configContext, { allowPartial: false })
+
+      const modules = graph.getModules()
+      const configContext = new ModuleConfigContext({
+        garden: this.garden,
+        resolvedProviders: providers,
+        variables: this.garden.variables,
+        dependencyConfigs: modules,
+        dependencyVersions: fromPairs(modules.map((m) => [m.name, m.version])),
+        runtimeContext,
+      })
+
+      // Set allowPartial=false to ensure all required strings are resolved.
+      service.config = resolveTemplateStrings(service.config, configContext, { allowPartial: false })
     }
 
     const handlerParams = {
@@ -822,7 +835,7 @@ export class ActionRouter implements TypeGuard {
   }) {
     let { task, log } = params
     const runtimeContext = params["runtimeContext"] as RuntimeContext | undefined
-    let module = omit(task.module, ["_ConfigType"])
+    let module = omit(task.module, ["_config"])
 
     log.silly(`Getting ${actionType} handler for task ${module.name}.${task.name}`)
 
@@ -836,13 +849,24 @@ export class ActionRouter implements TypeGuard {
     // Resolve ${runtime.*} template strings if needed.
     if (runtimeContext && (await getRuntimeTemplateReferences(module)).length > 0) {
       log.silly(`Resolving runtime template strings for task '${task.name}'`)
-      const configContext = await this.garden.getModuleConfigContext(runtimeContext)
-      // We first allow partial resolution on the full config graph, and then resolve the task config itself
-      // below with allowPartial=false to ensure all required strings are resolved.
-      const graph = await this.garden.getConfigGraph(log, { configContext, allowPartial: true })
-      task = await graph.getTask(task.name)
+
+      const providers = await this.garden.resolveProviders()
+      const graph = await this.garden.getConfigGraph(log, runtimeContext)
+      task = graph.getTask(task.name)
       module = task.module
-      task.config = await resolveTemplateStrings(task.config, configContext, { allowPartial: false })
+
+      const modules = graph.getModules()
+      const configContext = new ModuleConfigContext({
+        garden: this.garden,
+        resolvedProviders: providers,
+        variables: this.garden.variables,
+        dependencyConfigs: modules,
+        dependencyVersions: fromPairs(modules.map((m) => [m.name, m.version])),
+        runtimeContext,
+      })
+
+      // Set allowPartial=false to ensure all required strings are resolved.
+      task.config = resolveTemplateStrings(task.config, configContext, { allowPartial: false })
     }
 
     const handlerParams: any = {

garden-service/src/build-dir.ts

@@ -122,7 +122,7 @@ export class BuildDir {
         return
       }
 
-      const sourceModule = await graph.getModule(getModuleKey(buildDepConfig.name, buildDepConfig.plugin), true)
+      const sourceModule = graph.getModule(getModuleKey(buildDepConfig.name, buildDepConfig.plugin), true)
       const sourceBuildPath = await this.buildPath(sourceModule)
 
       // Sync to the module's top-level dir by default.

garden-service/src/commands/build.ts

@@ -84,16 +84,16 @@ export class BuildCommand extends Command<Args, Opts> {
     await garden.clearBuilds()
 
     const graph = await garden.getConfigGraph(log)
-    const modules = await graph.getModules({ names: args.modules })
+    const modules = graph.getModules({ names: args.modules })
     const moduleNames = modules.map((m) => m.name)
 
     const initialTasks = flatten(
-      await Bluebird.map(modules, (module) => BuildTask.factory({ garden, log, module, force: opts.force }))
+      await Bluebird.map(modules, (module) => BuildTask.factory({ garden, graph, log, module, force: opts.force }))
     )
 
     const results = await processModules({
       garden,
-      graph: await garden.getConfigGraph(log),
+      graph,
       log,
       footerLog,
       modules,
@@ -104,7 +104,7 @@ export class BuildCommand extends Command<Args, Opts> {
         const tasks = [module]
           .concat(deps.build)
           .filter((m) => moduleNames.includes(m.name))
-          .map((m) => BuildTask.factory({ garden, log, module: m, force: true }))
+          .map((m) => BuildTask.factory({ garden, graph, log, module: m, force: true }))
         return flatten(await Promise.all(tasks))
       },
     })

garden-service/src/commands/call.ts

@@ -66,7 +66,7 @@ export class CallCommand extends Command<Args> {
 
     // TODO: better error when service doesn't exist
     const graph = await garden.getConfigGraph(log)
-    const service = await graph.getService(serviceName)
+    const service = graph.getService(serviceName)
     // No need for full context, since we're just checking if the service is running.
     const runtimeContext = emptyRuntimeContext
     const actions = await garden.getActionRouter()

garden-service/src/commands/delete.ts

@@ -132,7 +132,7 @@ export class DeleteServiceCommand extends Command {
 
   async action({ garden, log, headerLog, args }: CommandParams<DeleteServiceArgs>): Promise<CommandResult> {
     const graph = await garden.getConfigGraph(log)
-    const services = await graph.getServices({ names: args.services })
+    const services = graph.getServices({ names: args.services })
 
     if (services.length === 0) {
       log.warn({ msg: "No services found. Aborting." })

garden-service/src/commands/dev.ts

@@ -114,7 +114,7 @@ export class DevCommand extends Command<DevCommandArgs, DevCommandOpts> {
     this.server.setGarden(garden)
 
     const graph = await garden.getConfigGraph(log)
-    const modules = await graph.getModules()
+    const modules = graph.getModules()
 
     const skipTests = opts["skip-tests"]
 
@@ -188,6 +188,7 @@ export async function getDevCommandInitialTasks({
       // Build the module (in case there are no tests, tasks or services here that need to be run)
       const buildTasks = await BuildTask.factory({
         garden,
+        graph,
         log,
         module,
         force: false,
@@ -207,7 +208,7 @@ export async function getDevCommandInitialTasks({
           })
 
       // Deploy all enabled services in module
-      const services = await graph.getServices({ names: module.serviceNames, includeDisabled: true })
+      const services = graph.getServices({ names: module.serviceNames, includeDisabled: true })
       const deployTasks = services
         .filter((s) => !s.disabled)
         .map(

garden-service/src/commands/exec.ts

@@ -76,7 +76,7 @@ export class ExecCommand extends Command<Args> {
     )
 
     const graph = await garden.getConfigGraph(log)
-    const service = await graph.getService(serviceName)
+    const service = graph.getService(serviceName)
     const actions = await garden.getActionRouter()
     const result = await actions.execInService({
       log,

garden-service/src/commands/get/get-task-result.ts

@@ -44,7 +44,7 @@ export class GetTaskResultCommand extends Command<Args> {
     const taskName = args.name
 
     const graph: ConfigGraph = await garden.getConfigGraph(log)
-    const task = await graph.getTask(taskName)
+    const task = graph.getTask(taskName)
 
     const actions = await garden.getActionRouter()
 

garden-service/src/commands/get/get-tasks.ts

@@ -62,9 +62,9 @@ export class GetTasksCommand extends Command<Args> {
 
   async action({ args, garden, log }: CommandParams<Args>): Promise<CommandResult> {
     const graph = await garden.getConfigGraph(log)
-    const tasks = await graph.getTasks({ names: args.tasks })
+    const tasks = graph.getTasks({ names: args.tasks })
     const taskModuleNames = uniq(tasks.map((t) => t.module.name))
-    const modules = sortBy(await graph.getModules({ names: taskModuleNames }), (m) => m.name)
+    const modules = sortBy(graph.getModules({ names: taskModuleNames }), (m) => m.name)
 
     const taskListing: any[] = []
     let logStr = ""

garden-service/src/commands/get/get-test-result.ts

@@ -58,7 +58,7 @@ export class GetTestResultCommand extends Command<Args> {
     const graph = await garden.getConfigGraph(log)
     const actions = await garden.getActionRouter()
 
-    const module = await graph.getModule(moduleName)
+    const module = graph.getModule(moduleName)
 
     const testConfig = findByName(module.testConfigs, testName)
 

garden-service/src/commands/link/module.ts

@@ -51,11 +51,14 @@ export class LinkModuleCommand extends Command<Args> {
 
     const { module: moduleName, path } = args
     const graph = await garden.getConfigGraph(log)
-    const moduleToLink = await graph.getModule(moduleName)
+    const moduleToLink = graph.getModule(moduleName)
 
     const isRemote = [moduleToLink].filter(hasRemoteSource)[0]
     if (!isRemote) {
-      const modulesWithRemoteSource = (await graph.getModules()).filter(hasRemoteSource).sort()
+      const modulesWithRemoteSource = graph
+        .getModules()
+        .filter(hasRemoteSource)
+        .sort()
 
       throw new ParameterError(
         `Expected module(s) ${chalk.underline(moduleName)} to have a remote source.` +

garden-service/src/commands/logs.ts

@@ -68,7 +68,7 @@ export class LogsCommand extends Command<Args, Opts> {
   async action({ garden, log, args, opts }: CommandParams<Args, Opts>): Promise<CommandResult<ServiceLogEntry[]>> {
     const { follow, tail } = opts
     const graph = await garden.getConfigGraph(log)
-    const services = await graph.getServices({ names: args.services })
+    const services = graph.getServices({ names: args.services })
     const serviceNames = services.map((s) => s.name).filter(Boolean)
     const maxServiceName = (maxBy(serviceNames, (serviceName) => serviceName.length) || "").length
 

garden-service/src/commands/plugins.ts

@@ -109,7 +109,7 @@ export class PluginsCommand extends Command<Args> {
     // Commands can optionally ask for all the modules in the project/environment
     if (command.resolveModules) {
       const graph = await garden.getConfigGraph(garden.log)
-      modules = await graph.getModules()
+      modules = graph.getModules()
     }
 
     log.info("")

garden-service/src/commands/publish.ts

@@ -14,6 +14,7 @@ import { Garden } from "../garden"
 import { LogEntry } from "../logger/log-entry"
 import { printHeader } from "../logger/util"
 import dedent = require("dedent")
+import { ConfigGraph } from "../config-graph"
 
 const publishArgs = {
   modules: new StringsParameter({
@@ -65,27 +66,42 @@ export class PublishCommand extends Command<Args, Opts> {
     printHeader(headerLog, "Publish modules", "rocket")
 
     const graph = await garden.getConfigGraph(log)
-    const modules = await graph.getModules({ names: args.modules })
+    const modules = graph.getModules({ names: args.modules })
 
-    const results = await publishModules(garden, log, modules, !!opts["force-build"], !!opts["allow-dirty"])
+    const results = await publishModules({
+      garden,
+      graph,
+      log,
+      modules,
+      forceBuild: !!opts["force-build"],
+      allowDirty: !!opts["allow-dirty"],
+    })
 
     return handleProcessResults(footerLog, "publish", { taskResults: results })
   }
 }
 
-export async function publishModules(
-  garden: Garden,
-  log: LogEntry,
-  modules: Module<any>[],
-  forceBuild: boolean,
+export async function publishModules({
+  garden,
+  graph,
+  log,
+  modules,
+  forceBuild,
+  allowDirty,
+}: {
+  garden: Garden
+  graph: ConfigGraph
+  log: LogEntry
+  modules: Module<any>[]
+  forceBuild: boolean
   allowDirty: boolean
-): Promise<TaskResults> {
+}): Promise<TaskResults> {
   if (!!allowDirty) {
     log.warn(`The --allow-dirty flag has been deprecated. It no longer has an effect.`)
   }
 
   const tasks = modules.map((module) => {
-    return new PublishTask({ garden, log, module, forceBuild })
+    return new PublishTask({ garden, graph, log, module, forceBuild })
   })
 
   return await garden.processTasks(tasks)

garden-service/src/commands/run/module.ts

@@ -84,7 +84,7 @@ export class RunModuleCommand extends Command<Args, Opts> {
     const moduleName = args.module
 
     const graph = await garden.getConfigGraph(log)
-    const module = await graph.getModule(moduleName)
+    const module = graph.getModule(moduleName)
 
     const msg = args.arguments
       ? `Running module ${chalk.white(moduleName)} with arguments ${chalk.white(args.arguments.join(" "))}`
@@ -96,13 +96,14 @@ export class RunModuleCommand extends Command<Args, Opts> {
 
     const buildTasks = await BuildTask.factory({
       garden,
+      graph,
       log,
       module,
       force: opts["force-build"],
     })
     await garden.processTasks(buildTasks)
 
-    const dependencies = await graph.getDependencies({ nodeType: "build", name: module.name, recursive: false })
+    const dependencies = graph.getDependencies({ nodeType: "build", name: module.name, recursive: false })
     const interactive = opts.interactive
 
     const runtimeContext = await prepareRuntimeContext({