diff --git a/core/src/garden.ts b/core/src/garden.ts index dee804c215..4c1f087ccc 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -45,8 +45,8 @@ import { BaseTask } from "./tasks/base" import { LocalConfigStore, ConfigStore, GlobalConfigStore, LinkedSource } from "./config-store" import { getLinkedSources, ExternalSourceType } from "./util/ext-source-util" import { BuildDependencyConfig, ModuleConfig } from "./config/module" -import { resolveModuleConfig } from "./resolve-module" -import { ModuleConfigContext, OutputConfigContext, DefaultEnvironmentContext } from "./config/config-context" +import { ModuleResolver, moduleResolutionConcurrencyLimit } from "./resolve-module" +import { OutputConfigContext, DefaultEnvironmentContext } from "./config/config-context" import { createPluginContext, CommandInfo } from "./plugin-context" import { ModuleAndRuntimeActionHandlers, RegisterPluginParam } from "./types/plugin/plugin" import { SUPPORTED_PLATFORMS, SupportedPlatform, DEFAULT_GARDEN_DIR_NAME } from "./constants" @@ -74,9 +74,8 @@ import { RuntimeContext } from "./runtime-context" import { loadPlugins, getDependencyOrder, getModuleTypes } from "./plugins" import { deline, naturalList } from "./util/string" import { ensureConnected } from "./db/connection" -import { cyclesToString, DependencyValidationGraph } from "./util/validate-dependencies" +import { DependencyValidationGraph } from "./util/validate-dependencies" import { Profile } from "./util/profiling" -import { ResolveModuleTask, getResolvedModules, moduleResolutionConcurrencyLimit } from "./tasks/resolve-module" import username from "username" import { throwOnMissingSecretKeys, resolveTemplateString } from "./template-string" import { WorkflowConfig, WorkflowConfigMap, resolveWorkflowConfig } from "./config/workflow" @@ -752,48 +751,21 @@ export class Garden { * For long-running processes, you need to call this again when any module or configuration has been updated. */ async getConfigGraph(log: LogEntry, runtimeContext?: RuntimeContext) { - const providers = await this.resolveProviders(log) - const configs = await this.getRawModuleConfigs() - log.silly(`Resolving module configs`) - // Resolve the project module configs - const tasks = configs.map( - (moduleConfig) => - new ResolveModuleTask({ garden: this, log, moduleConfig, resolvedProviders: providers, runtimeContext }) - ) - - let results: GraphResults - - try { - results = await this.processTasks(tasks) - } catch (err) { - // Wrap the circular dependency error to print a more specific message - if (err.type === "circular-dependencies") { - // Get the module names from the cycle keys (anything else is internal detail as far as users are concerned) - const cycleDescription = cyclesToString([err.detail.cycle.map((key: string) => key.split(".")[1])]) - throw new ConfigurationError( - dedent` - Detected circular dependencies between module configurations: - - ${cycleDescription} - `, - { cycle: err.detail.cycle } - ) - } else { - throw err - } - } + const resolvedProviders = await this.resolveProviders(log) + const rawConfigs = await this.getRawModuleConfigs() - const failed = Object.values(results).filter((r) => r?.error) + log.silly(`Resolving module configs`) - if (failed.length > 0) { - const errors = failed.map((r) => `${chalk.white.bold(r!.name)}: ${r?.error?.message}`) + // Resolve the project module configs + const resolver = new ModuleResolver({ + garden: this, + log, + rawConfigs, + resolvedProviders, + runtimeContext, + }) - throw new ConfigurationError(chalk.red(`Failed resolving one or more modules:\n\n${errors.join("\n")}`), { - results, - errors, - }) - } - const resolvedModules = getResolvedModules(results) + const resolvedModules = await resolver.resolveAll() const actions = await this.getActionRouter() const moduleTypes = await this.getModuleTypes() @@ -808,7 +780,7 @@ export class Garden { } // Walk through all plugins in dependency order, and allow them to augment the graph - const providerConfigs = Object.values(providers).map((p) => p.config) + const providerConfigs = Object.values(resolvedProviders).map((p) => p.config) for (const provider of getDependencyOrder(providerConfigs, this.registeredPlugins)) { // Skip the routine if the provider doesn't have the handler @@ -831,21 +803,10 @@ export class Garden { const { addBuildDependencies, addRuntimeDependencies, addModules } = await actions.augmentGraph({ pluginName: provider.name, log, - providers, + providers: resolvedProviders, modules: resolvedModules, }) - const configContext = new ModuleConfigContext({ - garden: this, - resolvedProviders: keyBy(providers, "name"), - dependencies: resolvedModules, - runtimeContext, - parentName: undefined, - templateName: undefined, - inputs: {}, - partialRuntimeResolution: true, - }) - // Resolve modules from specs and add to the list await Bluebird.map(addModules || [], async (spec) => { const path = spec.path || this.projectRoot @@ -854,7 +815,7 @@ export class Garden { // There is no actual config file for plugin modules (which the prepare function assumes) delete moduleConfig.configPath - const resolvedConfig = await resolveModuleConfig(this, moduleConfig, { configContext }) + const resolvedConfig = await resolver.resolveModuleConfig(moduleConfig, resolvedModules) resolvedModules.push(await moduleFromConfig(this, log, resolvedConfig, resolvedModules)) graph = undefined }) @@ -1176,7 +1137,7 @@ export class Garden { } public makeOverlapError(moduleOverlaps: ModuleOverlap[]) { - const overlapList = moduleOverlaps + const overlapList = sortBy(moduleOverlaps, (o) => o.module.name) .map(({ module, overlaps }) => { const formatted = overlaps.map((o) => { const detail = o.path === module.path ? "same path" : "nested" diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index 8aa5289d1a..15233a1ee1 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -6,131 +6,421 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { cloneDeep } from "lodash" +import { cloneDeep, keyBy } from "lodash" import { validateWithPath } from "./config/validation" -import { resolveTemplateStrings } from "./template-string" +import { resolveTemplateStrings, getModuleTemplateReferences, resolveTemplateString } from "./template-string" import { ContextResolveOpts, ModuleConfigContext } from "./config/config-context" -import { relative } from "path" +import { relative, resolve, posix } from "path" import { Garden } from "./garden" -import { ConfigurationError } from "./exceptions" -import { deline } from "./util/string" -import { getModuleKey } from "./types/module" +import { ConfigurationError, FilesystemError, PluginError } from "./exceptions" +import { deline, dedent } from "./util/string" +import { getModuleKey, ModuleConfigMap, GardenModule, ModuleMap, moduleFromConfig } from "./types/module" import { getModuleTypeBases } from "./plugins" import { ModuleConfig, moduleConfigSchema } from "./config/module" -import { profileAsync } from "./util/profiling" +import { Profile } from "./util/profiling" import { getLinkedSources } from "./util/ext-source-util" +import { ProviderMap } from "./config/provider" +import { RuntimeContext } from "./runtime-context" +import chalk from "chalk" +import { DependencyValidationGraph } from "./util/validate-dependencies" +import Bluebird from "bluebird" +import { readFile, mkdirp, writeFile } from "fs-extra" +import { LogEntry } from "./logger/log-entry" -export interface ModuleConfigResolveOpts extends ContextResolveOpts { - configContext: ModuleConfigContext -} +// This limit is fairly arbitrary, but we need to have some cap on concurrent processing. +export const moduleResolutionConcurrencyLimit = 40 -export const resolveModuleConfig = profileAsync(async function $resolveModuleConfig( - garden: Garden, - config: ModuleConfig, - opts: ModuleConfigResolveOpts -): Promise { - config = resolveTemplateStrings(cloneDeep(config), opts.configContext, opts) +/** + * Resolves a set of module configurations in dependency order. + * + * This operates differently than the TaskGraph in that it can add dependency links as it proceeds through the modules, + * which is important because dependencies can be discovered mid-stream, and the TaskGraph currently needs to + * statically resolve all dependencies before processing tasks. + */ +@Profile() +export class ModuleResolver { + private garden: Garden + private log: LogEntry + private rawConfigs: ModuleConfig[] + private rawConfigsByName: ModuleConfigMap + private resolvedProviders: ProviderMap + private runtimeContext?: RuntimeContext - const moduleTypeDefinitions = await garden.getModuleTypes() - const description = moduleTypeDefinitions[config.type] + constructor({ + garden, + log, + rawConfigs, + resolvedProviders, + runtimeContext, + }: { + garden: Garden + log: LogEntry + rawConfigs: ModuleConfig[] + resolvedProviders: ProviderMap + runtimeContext?: RuntimeContext + }) { + this.garden = garden + this.log = log + this.rawConfigs = rawConfigs + this.rawConfigsByName = keyBy(rawConfigs, "name") + this.resolvedProviders = resolvedProviders + this.runtimeContext = runtimeContext + } - if (!description) { - const configPath = relative(garden.projectRoot, config.configPath || config.path) + async resolveAll() { + // Collect template references for every raw config and work out module references in templates and explicit + // dependency references. We use two graphs, one will be fully populated as we progress, the other we gradually + // remove nodes from as we complete the processing. + const fullGraph = new DependencyValidationGraph() + const processingGraph = new DependencyValidationGraph() - throw new ConfigurationError( - deline` - Unrecognized module type '${config.type}' (defined at ${configPath}). - Are you missing a provider configuration? - `, - { config, configuredModuleTypes: Object.keys(moduleTypeDefinitions) } - ) + for (const rawConfig of this.rawConfigs) { + for (const graph of [fullGraph, processingGraph]) { + graph.addNode(rawConfig.name) + } + } + for (const rawConfig of this.rawConfigs) { + const deps = this.getModuleTemplateDependencies(rawConfig) + for (const graph of [fullGraph, processingGraph]) { + for (const dep of deps) { + graph.addNode(dep.name) + graph.addDependency(rawConfig.name, dep.name) + } + } + } + + const resolvedConfigs: ModuleConfigMap = {} + const resolvedModules: ModuleMap = {} + const errors: { [moduleName: string]: Error } = {} + + // Iterate through dependency graph, a batch of leaves at a time. While there are items remaining: + while (processingGraph.size() > 0) { + // Get batch of leaf nodes (ones with no unresolved dependencies). Implicitly checks for circular dependencies. + let batch: string[] + + try { + batch = processingGraph.overallOrder(true) + } catch (err) { + throw new ConfigurationError( + dedent` + Detected circular dependencies between module configurations: + + ${err.detail?.["circular-dependencies"] || err.message} + `, + { cycles: err.detail?.cycles } + ) + } + + // Process each of the leaf node module configs. + await Bluebird.map( + batch, + async (moduleName) => { + // Resolve configuration, unless previously resolved. + let resolvedConfig = resolvedConfigs[moduleName] + let foundNewDependency = false + + const dependencyNames = fullGraph.dependenciesOf(moduleName) + const resolvedDependencies = dependencyNames.map((n) => resolvedModules[n]) + + try { + if (!resolvedConfig) { + const rawConfig = this.rawConfigsByName[moduleName] + + resolvedConfig = resolvedConfigs[moduleName] = await this.resolveModuleConfig( + rawConfig, + resolvedDependencies + ) + + // Check if any new build dependencies were added by the configure handler + for (const dep of resolvedConfig.build.dependencies) { + if (!dependencyNames.includes(dep.name)) { + foundNewDependency = true + + // We throw if the build dependency can't be found at all + if (!fullGraph.hasNode(dep.name)) { + this.missingBuildDependency(moduleName, dep.name) + } + fullGraph.addDependency(moduleName, dep.name) + + // The dependency may already have been processed, we don't want to add it to the graph in that case + if (processingGraph.hasNode(dep.name)) { + processingGraph.addDependency(moduleName, dep.name) + } + } + } + } + + // If no build dependency was added, fully resolve the module and remove from graph, otherwise keep it + // in the graph and move on to make sure we fully resolve the dependencies and don't run into circular + // dependencies. + if (!foundNewDependency) { + resolvedModules[moduleName] = await this.resolveModule(resolvedConfig, resolvedDependencies) + processingGraph.removeNode(moduleName) + } + } catch (err) { + errors[moduleName] = err + } + }, + { concurrency: moduleResolutionConcurrencyLimit } + ) + + if (Object.keys(errors).length > 0) { + const errorStr = Object.entries(errors) + .map(([name, err]) => `${chalk.white.bold(name)}: ${err.message}`) + .join("\n") + + throw new ConfigurationError(chalk.red(`Failed resolving one or more modules:\n\n${errorStr}`), { + errors, + }) + } + } + + return Object.values(resolvedModules) } - // Validate the module-type specific spec - if (description.schema) { - config.spec = validateWithPath({ - config: config.spec, - configType: "Module", - schema: description.schema, - name: config.name, - path: config.path, - projectRoot: garden.projectRoot, + private getModuleTemplateDependencies(rawConfig: ModuleConfig) { + const configContext = new ModuleConfigContext({ + garden: this.garden, + resolvedProviders: this.resolvedProviders, + moduleName: rawConfig.name, + dependencies: [], + runtimeContext: this.runtimeContext, + parentName: rawConfig.parentName, + templateName: rawConfig.templateName, + inputs: rawConfig.inputs, + partialRuntimeResolution: true, }) - } - /* - We allow specifying modules by name only as a shorthand: + const templateRefs = getModuleTemplateReferences(rawConfig, configContext) + const deps = templateRefs.filter((d) => d[1] !== rawConfig.name) - dependencies: - - foo-module - - name: foo-module // same as the above - */ - if (config.build && config.build.dependencies) { - config.build.dependencies = config.build.dependencies.map((dep) => - typeof dep === "string" ? { name: dep, copy: [] } : dep + return deps.map((d) => { + const name = d[1] + const moduleConfig = this.rawConfigsByName[name] + + if (!moduleConfig) { + this.missingBuildDependency(rawConfig.name, name as string) + } + + return moduleConfig + }) + } + + private missingBuildDependency(moduleName: string, dependencyName: string) { + throw new ConfigurationError( + chalk.red( + `Could not find build dependency ${chalk.white(dependencyName)}, ` + + `configured in module ${chalk.white(moduleName)}` + ), + { moduleName, dependencyName } ) } - // Validate the base config schema - config = validateWithPath({ - config, - schema: moduleConfigSchema(), - configType: "module", - name: config.name, - path: config.path, - projectRoot: garden.projectRoot, - }) - - if (config.repositoryUrl) { - const linkedSources = await getLinkedSources(garden, "module") - config.path = await garden.loadExtSourcePath({ - name: config.name, - linkedSources, - repositoryUrl: config.repositoryUrl, - sourceType: "module", + /** + * Resolves and validates a single module configuration. + */ + async resolveModuleConfig(config: ModuleConfig, dependencies: GardenModule[]): Promise { + const garden = this.garden + const configContext = new ModuleConfigContext({ + garden: this.garden, + resolvedProviders: this.resolvedProviders, + moduleName: config.name, + dependencies, + runtimeContext: this.runtimeContext, + parentName: config.parentName, + templateName: config.templateName, + inputs: config.inputs, + partialRuntimeResolution: true, }) - } - const actions = await garden.getActionRouter() - const configureResult = await actions.configureModule({ - moduleConfig: config, - log: garden.log, - }) + config = resolveTemplateStrings(cloneDeep(config), configContext, { + allowPartial: false, + }) - config = configureResult.moduleConfig + const moduleTypeDefinitions = await this.garden.getModuleTypes() + const description = moduleTypeDefinitions[config.type] - // Validate the configure handler output against the module type's bases - const bases = getModuleTypeBases(moduleTypeDefinitions[config.type], moduleTypeDefinitions) + if (!description) { + const configPath = relative(garden.projectRoot, config.configPath || config.path) - for (const base of bases) { - if (base.schema) { - garden.log.silly(`Validating '${config.name}' config against '${base.name}' schema`) + throw new ConfigurationError( + deline` + Unrecognized module type '${config.type}' (defined at ${configPath}). + Are you missing a provider configuration? + `, + { config, configuredModuleTypes: Object.keys(moduleTypeDefinitions) } + ) + } - config.spec = validateWithPath({ + // Validate the module-type specific spec + if (description.schema) { + config.spec = validateWithPath({ config: config.spec, - schema: base.schema.unknown(true), - path: garden.projectRoot, + configType: "Module", + schema: description.schema, + name: config.name, + path: config.path, projectRoot: garden.projectRoot, - configType: `configuration for module '${config.name}' (base schema from '${base.name}' plugin)`, - ErrorClass: ConfigurationError, }) } - } - // FIXME: We should be able to avoid this - config.name = getModuleKey(config.name, config.plugin) + /* + We allow specifying modules by name only as a shorthand: + + dependencies: + - foo-module + - name: foo-module // same as the above + */ + if (config.build && config.build.dependencies) { + config.build.dependencies = config.build.dependencies.map((dep) => + typeof dep === "string" ? { name: dep, copy: [] } : dep + ) + } + + // Validate the base config schema + config = validateWithPath({ + config, + schema: moduleConfigSchema(), + configType: "module", + name: config.name, + path: config.path, + projectRoot: garden.projectRoot, + }) + + if (config.repositoryUrl) { + const linkedSources = await getLinkedSources(garden, "module") + config.path = await garden.loadExtSourcePath({ + name: config.name, + linkedSources, + repositoryUrl: config.repositoryUrl, + sourceType: "module", + }) + } + + const actions = await garden.getActionRouter() + const configureResult = await actions.configureModule({ + moduleConfig: config, + log: garden.log, + }) + + config = configureResult.moduleConfig + + // Validate the configure handler output against the module type's bases + const bases = getModuleTypeBases(moduleTypeDefinitions[config.type], moduleTypeDefinitions) - if (config.plugin) { - for (const serviceConfig of config.serviceConfigs) { - serviceConfig.name = getModuleKey(serviceConfig.name, config.plugin) + for (const base of bases) { + if (base.schema) { + garden.log.silly(`Validating '${config.name}' config against '${base.name}' schema`) + + config.spec = validateWithPath({ + config: config.spec, + schema: base.schema.unknown(true), + path: garden.projectRoot, + projectRoot: garden.projectRoot, + configType: `configuration for module '${config.name}' (base schema from '${base.name}' plugin)`, + ErrorClass: ConfigurationError, + }) + } } - for (const taskConfig of config.taskConfigs) { - taskConfig.name = getModuleKey(taskConfig.name, config.plugin) + + // FIXME: We should be able to avoid this + config.name = getModuleKey(config.name, config.plugin) + + if (config.plugin) { + for (const serviceConfig of config.serviceConfigs) { + serviceConfig.name = getModuleKey(serviceConfig.name, config.plugin) + } + for (const taskConfig of config.taskConfigs) { + taskConfig.name = getModuleKey(taskConfig.name, config.plugin) + } + for (const testConfig of config.testConfigs) { + testConfig.name = getModuleKey(testConfig.name, config.plugin) + } + } + + return config + } + + private async resolveModule(resolvedConfig: ModuleConfig, dependencies: GardenModule[]) { + // Write module files + const configContext = new ModuleConfigContext({ + garden: this.garden, + resolvedProviders: this.resolvedProviders, + moduleName: resolvedConfig.name, + dependencies, + runtimeContext: this.runtimeContext, + parentName: resolvedConfig.parentName, + templateName: resolvedConfig.templateName, + inputs: resolvedConfig.inputs, + partialRuntimeResolution: true, + }) + + await Bluebird.map(resolvedConfig.generateFiles || [], async (fileSpec) => { + let contents = fileSpec.value || "" + + if (fileSpec.sourcePath) { + contents = (await readFile(fileSpec.sourcePath)).toString() + contents = await resolveTemplateString(contents, configContext) + } + + const resolvedContents = resolveTemplateString(contents, configContext) + const targetDir = resolve(resolvedConfig.path, ...posix.dirname(fileSpec.targetPath).split(posix.sep)) + const targetPath = resolve(resolvedConfig.path, ...fileSpec.targetPath.split(posix.sep)) + + try { + await mkdirp(targetDir) + await writeFile(targetPath, resolvedContents) + } catch (error) { + throw new FilesystemError( + `Unable to write templated file ${fileSpec.targetPath} from ${resolvedConfig.name}: ${error.message}`, + { + fileSpec, + error, + } + ) + } + }) + + const module = await moduleFromConfig(this.garden, this.log, resolvedConfig, dependencies) + + const moduleTypeDefinitions = await this.garden.getModuleTypes() + const description = moduleTypeDefinitions[module.type]! + + // Validate the module outputs against the outputs schema + if (description.moduleOutputsSchema) { + module.outputs = validateWithPath({ + config: module.outputs, + schema: description.moduleOutputsSchema, + configType: `outputs for module`, + name: module.name, + path: module.configPath || module.path, + projectRoot: this.garden.projectRoot, + ErrorClass: PluginError, + }) } - for (const testConfig of config.testConfigs) { - testConfig.name = getModuleKey(testConfig.name, config.plugin) + + // Validate the module outputs against the module type's bases + const bases = getModuleTypeBases(moduleTypeDefinitions[module.type], moduleTypeDefinitions) + + for (const base of bases) { + if (base.moduleOutputsSchema) { + this.log.silly(`Validating '${module.name}' module outputs against '${base.name}' schema`) + + module.outputs = validateWithPath({ + config: module.outputs, + schema: base.moduleOutputsSchema.unknown(true), + path: module.configPath || module.path, + projectRoot: this.garden.projectRoot, + configType: `outputs for module '${module.name}' (base schema from '${base.name}' plugin)`, + ErrorClass: PluginError, + }) + } } + + return module } +} - return config -}) +export interface ModuleConfigResolveOpts extends ContextResolveOpts { + configContext: ModuleConfigContext +} diff --git a/core/src/tasks/resolve-module.ts b/core/src/tasks/resolve-module.ts deleted file mode 100644 index 2a0642b3ec..0000000000 --- a/core/src/tasks/resolve-module.ts +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Copyright (C) 2018-2020 Garden Technologies, Inc. - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - */ - -import chalk from "chalk" -import { GardenModule, moduleFromConfig, getModuleKey } from "../types/module" -import { BaseTask, TaskType } from "../tasks/base" -import { Garden } from "../garden" -import { LogEntry } from "../logger/log-entry" -import { ModuleConfig } from "../config/module" -import { GraphResults } from "../task-graph" -import { keyBy } from "lodash" -import { ConfigurationError, PluginError, FilesystemError } from "../exceptions" -import { RuntimeContext } from "../runtime-context" -import { ModuleConfigContext } from "../config/config-context" -import { ProviderMap } from "../config/provider" -import { resolveModuleConfig } from "../resolve-module" -import { getModuleTemplateReferences, resolveTemplateString } from "../template-string" -import { Profile } from "../util/profiling" -import { validateWithPath } from "../config/validation" -import { getModuleTypeBases } from "../plugins" -import Bluebird from "bluebird" -import { posix, resolve } from "path" -import { mkdirp, writeFile, readFile } from "fs-extra" - -interface ResolveModuleConfigTaskParams { - garden: Garden - log: LogEntry - moduleConfig: ModuleConfig - resolvedProviders: ProviderMap - runtimeContext?: RuntimeContext -} - -/** - * Resolve the module configuration, i.e. resolve all template strings and call the provider configure handler(s). - * If necessary, this may involve resolving dependencies fully (using the ModuleResolveTask, see below). - */ -@Profile() -export class ResolveModuleConfigTask extends BaseTask { - type: TaskType = "resolve-module-config" - - private moduleConfig: ModuleConfig - private resolvedProviders: ProviderMap - private runtimeContext?: RuntimeContext - - constructor({ garden, log, moduleConfig, resolvedProviders, runtimeContext }: ResolveModuleConfigTaskParams) { - super({ garden, log, force: true, version: garden.version }) - this.moduleConfig = moduleConfig - this.resolvedProviders = resolvedProviders - this.runtimeContext = runtimeContext - } - - async resolveDependencies() { - const rawConfigs = keyBy(await this.garden.getRawModuleConfigs(), "name") - - const configContext = new ModuleConfigContext({ - garden: this.garden, - resolvedProviders: this.resolvedProviders, - moduleName: this.moduleConfig.name, - dependencies: [], - runtimeContext: this.runtimeContext, - parentName: this.moduleConfig.parentName, - templateName: this.moduleConfig.templateName, - inputs: this.moduleConfig.inputs, - partialRuntimeResolution: true, - }) - - const templateRefs = getModuleTemplateReferences(this.moduleConfig, configContext) - const deps = templateRefs.filter((d) => d[1] !== this.moduleConfig.name) - - return deps.map((d) => { - const name = d[1] - const moduleConfig = rawConfigs[name] - - if (!moduleConfig) { - throw new ConfigurationError( - chalk.red( - `Could not find build dependency ${chalk.white(name)}, configured in module ${chalk.white( - this.moduleConfig.name - )}` - ), - { moduleConfig } - ) - } - - return new ResolveModuleTask({ - garden: this.garden, - log: this.log, - moduleConfig, - resolvedProviders: this.resolvedProviders, - runtimeContext: this.runtimeContext, - }) - }) - } - - getName() { - return this.moduleConfig.name - } - - getDescription() { - return `resolving module config ${this.getName()}` - } - - async process(dependencyResults: GraphResults): Promise { - const dependencies = getResolvedModules(dependencyResults) - - const configContext = new ModuleConfigContext({ - garden: this.garden, - resolvedProviders: this.resolvedProviders, - moduleName: this.moduleConfig.name, - dependencies, - runtimeContext: this.runtimeContext, - parentName: this.moduleConfig.parentName, - templateName: this.moduleConfig.templateName, - inputs: this.moduleConfig.inputs, - partialRuntimeResolution: true, - }) - - return resolveModuleConfig(this.garden, this.moduleConfig, { - allowPartial: false, - configContext, - }) - } -} - -interface ResolveModuleTaskParams { - garden: Garden - log: LogEntry - moduleConfig: ModuleConfig - resolvedProviders: ProviderMap - runtimeContext?: RuntimeContext -} - -export const moduleResolutionConcurrencyLimit = 40 - -/** - * Fully resolve the given module config, including its final version and dependencies. - */ -@Profile() -export class ResolveModuleTask extends BaseTask { - type: TaskType = "resolve-module" - - // It's advisable to have _some_ limit (say if you have hundreds of modules), because the filesystem scan can cost - // a bit of memory, but we make it quite a bit higher than other tasks. - concurrencyLimit = moduleResolutionConcurrencyLimit - - private moduleConfig: ModuleConfig - private resolvedProviders: ProviderMap - private runtimeContext?: RuntimeContext - - constructor({ garden, log, moduleConfig, resolvedProviders, runtimeContext }: ResolveModuleTaskParams) { - super({ garden, log, force: true, version: garden.version }) - this.moduleConfig = moduleConfig - this.resolvedProviders = resolvedProviders - this.runtimeContext = runtimeContext - } - - async resolveDependencies() { - const rawConfigs = keyBy(await this.garden.getRawModuleConfigs(), "name") - - const deps = this.moduleConfig.build.dependencies - .map((d) => getModuleKey(d.name, d.plugin)) - .map((key) => { - const moduleConfig = rawConfigs[key] - - if (!moduleConfig) { - throw new ConfigurationError( - chalk.red( - `Could not find build dependency ${chalk.white(key)}, configured in module ${chalk.white( - this.moduleConfig.name - )}` - ), - { moduleConfig } - ) - } - - return new ResolveModuleTask({ - garden: this.garden, - log: this.log, - moduleConfig, - resolvedProviders: this.resolvedProviders, - runtimeContext: this.runtimeContext, - }) - }) - - return [ - // Need to resolve own config - new ResolveModuleConfigTask({ - garden: this.garden, - log: this.log, - moduleConfig: this.moduleConfig, - resolvedProviders: this.resolvedProviders, - runtimeContext: this.runtimeContext, - }), - // As well as all the module's build dependencies - ...deps, - ] - } - - getName() { - return this.moduleConfig.name - } - - getDescription() { - return `resolving module ${this.getName()}` - } - - async process(dependencyResults: GraphResults): Promise { - const resolvedConfig = dependencyResults["resolve-module-config." + this.getName()]!.output as ModuleConfig - const dependencyModules = getResolvedModules(dependencyResults) - - // Write module files - const configContext = new ModuleConfigContext({ - garden: this.garden, - resolvedProviders: this.resolvedProviders, - moduleName: this.moduleConfig.name, - dependencies: dependencyModules, - runtimeContext: this.runtimeContext, - parentName: this.moduleConfig.parentName, - templateName: this.moduleConfig.templateName, - inputs: this.moduleConfig.inputs, - partialRuntimeResolution: true, - }) - - await Bluebird.map(resolvedConfig.generateFiles || [], async (fileSpec) => { - let contents = fileSpec.value || "" - - if (fileSpec.sourcePath) { - contents = (await readFile(fileSpec.sourcePath)).toString() - contents = await resolveTemplateString(contents, configContext) - } - - const resolvedContents = resolveTemplateString(contents, configContext) - const targetDir = resolve(resolvedConfig.path, ...posix.dirname(fileSpec.targetPath).split(posix.sep)) - const targetPath = resolve(resolvedConfig.path, ...fileSpec.targetPath.split(posix.sep)) - - try { - await mkdirp(targetDir) - await writeFile(targetPath, resolvedContents) - } catch (error) { - throw new FilesystemError( - `Unable to write templated file ${fileSpec.targetPath} from ${resolvedConfig.name}: ${error.message}`, - { - fileSpec, - error, - } - ) - } - }) - - const module = await moduleFromConfig(this.garden, this.log, resolvedConfig, dependencyModules) - - const moduleTypeDefinitions = await this.garden.getModuleTypes() - const description = moduleTypeDefinitions[module.type]! - - // Validate the module outputs against the outputs schema - if (description.moduleOutputsSchema) { - module.outputs = validateWithPath({ - config: module.outputs, - schema: description.moduleOutputsSchema, - configType: `outputs for module`, - name: module.name, - path: module.configPath || module.path, - projectRoot: this.garden.projectRoot, - ErrorClass: PluginError, - }) - } - - // Validate the module outputs against the module type's bases - const bases = getModuleTypeBases(moduleTypeDefinitions[module.type], moduleTypeDefinitions) - - for (const base of bases) { - if (base.moduleOutputsSchema) { - this.log.silly(`Validating '${module.name}' module outputs against '${base.name}' schema`) - - module.outputs = validateWithPath({ - config: module.outputs, - schema: base.moduleOutputsSchema.unknown(true), - path: module.configPath || module.path, - projectRoot: this.garden.projectRoot, - configType: `outputs for module '${module.name}' (base schema from '${base.name}' plugin)`, - ErrorClass: PluginError, - }) - } - } - - return module - } -} - -export function getResolvedModules(dependencyResults: GraphResults): GardenModule[] { - return Object.values(dependencyResults) - .filter((r) => r && r.type === "resolve-module") - .map((r) => r!.output) as GardenModule[] -} diff --git a/core/src/util/validate-dependencies.ts b/core/src/util/validate-dependencies.ts index 7c8e910b9e..cfabbc7d62 100644 --- a/core/src/util/validate-dependencies.ts +++ b/core/src/util/validate-dependencies.ts @@ -11,7 +11,7 @@ import { flatten, uniq } from "lodash" import indentString from "indent-string" import { get, isEqual, join, set, uniqWith } from "lodash" import { getModuleKey } from "../types/module" -import { ConfigurationError, ParameterError } from "../exceptions" +import { ConfigurationError } from "../exceptions" import { ModuleConfig } from "../config/module" import { deline } from "./string" import { DependencyGraph, DependencyGraphNode, nodeKey as configGraphNodeKey } from "../config-graph" @@ -72,14 +72,11 @@ export type DependencyValidationGraphNode = { description?: string // used instead of key when rendering node in circular dependency error messages } +/** + * Extends the dependency-graph module to improve circular dependency detection (see below). + */ @Profile() -export class DependencyValidationGraph { - graph: { [nodeKey: string]: DependencyValidationGraphNode } - - constructor(nodes?: DependencyValidationGraphNode[]) { - this.graph = Object.fromEntries((nodes || []).map((n) => [n.key, n])) - } - +export class DependencyValidationGraph extends DepGraph { static fromDependencyGraph(dependencyGraph: DependencyGraph) { const withDeps = (node: DependencyGraphNode): DependencyValidationGraphNode => { return { @@ -87,68 +84,31 @@ export class DependencyValidationGraph { dependencies: node.dependencies.map((d) => configGraphNodeKey(d.type, d.name)), } } + + const graph = new DependencyValidationGraph() const nodes = Object.values(dependencyGraph).map((n) => withDeps(n)) - return new DependencyValidationGraph(nodes) - } - overallOrder(): string[] { - const cycles = this.detectCircularDependencies() - if (cycles.length > 0) { - const description = cyclesToString(cycles) - const errMsg = `\nCircular dependencies detected: \n\n${description}\n` - throw new ConfigurationError(errMsg, { "circular-dependencies": description }) + for (const node of nodes || []) { + graph.addNode(node.key, node.description) } - - const depGraph = new DepGraph() - for (const node of Object.values(this.graph)) { - depGraph.addNode(node.key) + for (const node of nodes || []) { for (const dep of node.dependencies) { - depGraph.addNode(dep) - depGraph.addDependency(node.key, dep) + graph.addDependency(node.key, dep) } } - return depGraph.overallOrder() - } - /** - * Idempotent. - * - * If provided, description will be used instead of key when rendering the node in - * circular dependency error messages. - */ - addNode(key: string, description?: string) { - if (!this.graph[key]) { - this.graph[key] = { key, dependencies: [], description } - } + return graph } - /** - * Idempotent. - * - * Throws an error if a node doesn't exist for either dependantKey or dependencyKey. - */ - addDependency(dependantKey: string, dependencyKey: string) { - if (!this.graph[dependantKey]) { - throw new ParameterError(`addDependency: no node exists for dependantKey ${dependantKey}`, { - dependantKey, - dependencyKey, - graph: this.graph, - }) - } - - if (!this.graph[dependencyKey]) { - throw new ParameterError(`addDependency: no node exists for dependencyKey ${dependencyKey}`, { - dependantKey, - dependencyKey, - graph: this.graph, - }) + overallOrder(leavesOnly?: boolean): string[] { + const cycles = this.detectCircularDependencies() + if (cycles.length > 0) { + const description = cyclesToString(cycles) + const errMsg = `\nCircular dependencies detected: \n\n${description}\n` + throw new ConfigurationError(errMsg, { "circular-dependencies": description, cycles }) } - const dependant = this.graph[dependantKey] - if (!dependant.dependencies.find((d) => d === dependencyKey)) { - const dependency = this.graph[dependencyKey] - dependant.dependencies.push(dependency.key) - } + return super.overallOrder(leavesOnly) } /** @@ -157,9 +117,9 @@ export class DependencyValidationGraph { detectCircularDependencies(): Cycle[] { const edges: DependencyEdge[] = [] - for (const node of Object.values(this.graph)) { - for (const dep of node.dependencies) { - edges.push({ from: node.key, to: dep }) + for (const [node, deps] of Object.entries(this["outgoingEdges"])) { + for (const dep of deps) { + edges.push({ from: node, to: dep }) } } @@ -168,7 +128,7 @@ export class DependencyValidationGraph { cyclesToString(cycles: Cycle[]) { const cycleDescriptions = cycles.map((c) => { - const nodeDescriptions = c.map((key) => this.graph[key].description || key) + const nodeDescriptions = c.map((key) => this["nodes"][key] || key) return join(nodeDescriptions.concat([nodeDescriptions[0]]), " <- ") }) return cycleDescriptions.length === 1 ? cycleDescriptions[0] : cycleDescriptions.join("\n\n") diff --git a/core/test/data/test-projects/1067-module-ref-within-file/garden.yml b/core/test/data/test-projects/1067-module-ref-within-file/garden.yml index 5715b783d1..79609dce74 100644 --- a/core/test/data/test-projects/1067-module-ref-within-file/garden.yml +++ b/core/test/data/test-projects/1067-module-ref-within-file/garden.yml @@ -18,8 +18,6 @@ allowPublish: false include: ["*"] name: exec-module build: - dependencies: - - name: container-module command: - echo - "${modules.container-module.path}" diff --git a/core/test/unit/src/commands/dev.ts b/core/test/unit/src/commands/dev.ts index 6294be3e4a..3ea9646e95 100644 --- a/core/test/unit/src/commands/dev.ts +++ b/core/test/unit/src/commands/dev.ts @@ -97,9 +97,6 @@ describe("DevCommand", () => { "get-service-status.service-b", "get-service-status.service-c", "get-task-result.task-c", - "resolve-module-config.module-a", - "resolve-module-config.module-b", - "resolve-module-config.module-c", "resolve-provider.container", "resolve-provider.exec", "resolve-provider.templated", diff --git a/core/test/unit/src/config-graph.ts b/core/test/unit/src/config-graph.ts index 74c345766b..8931e3c4b7 100644 --- a/core/test/unit/src/config-graph.ts +++ b/core/test/unit/src/config-graph.ts @@ -125,7 +125,7 @@ describe("ConfigGraph", () => { it("should throw if named module is missing", async () => { try { - await graphA.getModules({ names: ["bla"] }) + graphA.getModules({ names: ["bla"] }) } catch (err) { expect(err.type).to.equal("parameter") return diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index 0f97a0a6b6..19918d4a0e 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -2713,6 +2713,77 @@ describe("Garden", () => { expect(module.spec.bla).to.eql({ a: "a", b: "B", c: "c" }) }) + it("should correctly handle build dependencies added by module configure handlers", async () => { + const test = createGardenPlugin({ + name: "test", + createModuleTypes: [ + { + name: "test", + docs: "test", + schema: joi.object(), + handlers: { + async configure({ moduleConfig }) { + if (moduleConfig.name === "module-b") { + moduleConfig.build.dependencies = [{ name: "module-a", copy: [] }] + } + return { moduleConfig } + }, + }, + }, + ], + }) + + const garden = await TestGarden.factory(pathFoo, { + plugins: [test], + config: { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "test", + path: pathFoo, + defaultEnvironment: "default", + dotIgnoreFiles: [], + environments: [{ name: "default", defaultNamespace, variables: {} }], + providers: [{ name: "test" }], + variables: {}, + }, + }) + + garden.setModuleConfigs([ + { + apiVersion: DEFAULT_API_VERSION, + name: "module-a", + type: "test", + allowPublish: false, + build: { dependencies: [] }, + disabled: false, + include: [], + path: pathFoo, + serviceConfigs: [], + taskConfigs: [], + testConfigs: [], + spec: {}, + }, + { + apiVersion: DEFAULT_API_VERSION, + name: "module-b", + type: "test", + allowPublish: false, + build: { dependencies: [] }, + disabled: false, + include: [], + path: pathFoo, + serviceConfigs: [], + taskConfigs: [], + testConfigs: [], + spec: {}, + }, + ]) + + const module = await garden.resolveModule("module-b") + + expect(module.buildDependencies["module-a"]?.name).to.equal("module-a") + }) + it("should handle module references within single file", async () => { const projectRoot = getDataDir("test-projects", "1067-module-ref-within-file") const garden = await makeTestGarden(projectRoot)