Skip to content

Commit

Permalink
perf: extend partial resolution to action graph resolution
Browse files Browse the repository at this point in the history
  • Loading branch information
@edvald @thsig
edvald authored and thsig committed May 8, 2024
1 parent a502a31 commit 2785f95
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 32 deletions.
1 change: 1 addition & 0 deletions core/src/garden.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1081,6 +1081,7 @@ export class Garden {
moduleGraph,
actionModes,
linkedSources,
actionsFilter,
})

// Walk through all plugins in dependency order, and allow them to augment the graph
Expand Down
125 changes: 93 additions & 32 deletions core/src/graph/actions.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ import {
resolveTemplateStrings,
} from "../template-string/template-string.js"
import { dedent, deline, naturalList } from "../util/string.js"
import { getVarfileData, mergeVariables } from "./common.js"
import { getVarfileData, DependencyGraph, mergeVariables } from "./common.js"
import type { ConfigGraph } from "./config-graph.js"
import { MutableConfigGraph } from "./config-graph.js"
import type { ModuleGraph } from "./modules.js"
Expand All @@ -78,6 +78,7 @@ export const actionConfigsToGraph = profileAsync(async function actionConfigsToG
moduleGraph,
actionModes,
linkedSources,
actionsFilter,
}: {
garden: Garden
log: Log
Expand All @@ -86,6 +87,7 @@ export const actionConfigsToGraph = profileAsync(async function actionConfigsToG
moduleGraph: ModuleGraph
actionModes: ActionModeMap
linkedSources: LinkedSourceMap
actionsFilter?: string[]
}): Promise<MutableConfigGraph> {
const configsByKey: ActionConfigsByKey = {}

Expand Down Expand Up @@ -146,9 +148,14 @@ export const actionConfigsToGraph = profileAsync(async function actionConfigsToG

const { mode, explicitMode } = getActionMode(config, actionModes, log)
computedActionModes[key] = { mode, explicitMode }
const actionTypes = await garden.getActionTypes()
const definition = actionTypes[config.kind][config.type]?.spec
preprocessResults[key] = await preprocessActionConfig({
garden,
config,
configsByKey,
actionTypes,
definition,
router,
linkedSources,
log,
Expand All @@ -166,22 +173,16 @@ export const actionConfigsToGraph = profileAsync(async function actionConfigsToG
//
// The proper solution to this would involve e.g. parametrized actions, or injecting a separate Build action
// with `this.mode` set to the Deploy action's mode before resolution (both would need to be thought out carefully).
const actionTypes = await garden.getActionTypes()
const buildModeOverrides: Record<string, { mode: ActionMode; overriddenByDeploy: string }> = {}

for (const [key, res] of Object.entries(preprocessResults)) {
const config = res.config
const { mode } = computedActionModes[key]

if (config.kind === "Deploy" && mode !== "default") {
const definition = actionTypes[config.kind][config.type]?.spec
const buildDeps = dependenciesFromActionConfig(
log,
config,
configsByKey,
definition,
res.templateContext,
actionTypes
)
const buildDeps = res.dependencies.filter((d) => d.kind === "Build")
const referencedBuildNames = [config.build, ...buildDeps.map((d) => d.name)].filter(isTruthy)

for (const buildName of referencedBuildNames) {
const buildKey = actionReferenceToString({ kind: "Build", name: buildName })
actionModes[mode] = [buildKey, ...(actionModes[mode] || [])]
Expand All @@ -197,6 +198,45 @@ export const actionConfigsToGraph = profileAsync(async function actionConfigsToG
// We are preprocessing actions in two batches so we can infer the mode of Build actions that are used by Deploy actions. See the comments above.
await preprocessActions((config) => config.kind !== "Deploy")

// Apply actionsFilter if provided to avoid unnecessary VCS scanning and resolution
if (actionsFilter) {
const depGraph = new DependencyGraph<string>()

for (const [key, res] of Object.entries(preprocessResults)) {
const { dependencies } = res

depGraph.addNode(key)

for (const dep of dependencies) {
const depKey = actionReferenceToString(dep)
depGraph.addNode(depKey)
depGraph.addDependency(key, depKey)
}
}

const requiredKeys = new Set<string>()

const matched = Object.keys(preprocessResults).filter((key) =>
actionsFilter.some((pattern) => minimatch(key, pattern))
)

for (const key of matched) {
// Matches a filter
requiredKeys.add(key)

// Also keep all dependencies of matched actions, transitively
for (const depKey of depGraph.dependenciesOf(key)) {
requiredKeys.add(depKey)
}
}

for (const key of Object.keys(preprocessResults)) {
if (!requiredKeys.has(key)) {
delete preprocessResults[key]
}
}
}

// Optimize file scanning by avoiding unnecessarily broad scans when project is not in repo root.
const preprocessedConfigs = Object.values(preprocessResults).map((r) => r.config)
const allPaths = preprocessedConfigs.map((c) => getSourcePath(c))
Expand All @@ -212,20 +252,19 @@ export const actionConfigsToGraph = profileAsync(async function actionConfigsToG

await Promise.all(
Object.entries(preprocessResults).map(async ([key, res]) => {
const { config, linkedSource, remoteSourcePath, supportedModes, templateContext } = res
const { config, linkedSource, remoteSourcePath, supportedModes, dependencies } = res
const { mode, explicitMode } = computedActionModes[key]

try {
const action = await processActionConfig({
garden,
graph,
config,
dependencies,
log,
configsByKey,
mode,
linkedSource,
remoteSourcePath,
templateContext,
supportedModes,
scanRoot: minimalRoots[getSourcePath(config)],
})
Expand Down Expand Up @@ -317,25 +356,29 @@ export const actionFromConfig = profileAsync(async function actionFromConfig({
scanRoot?: string
}) {
// Call configure handler and validate
const { config, supportedModes, linkedSource, remoteSourcePath, templateContext } = await preprocessActionConfig({
const actionTypes = await garden.getActionTypes()
const definition = actionTypes[inputConfig.kind][inputConfig.type]?.spec
const { config, supportedModes, linkedSource, remoteSourcePath, dependencies } = await preprocessActionConfig({
garden,
config: inputConfig,
actionTypes,
definition,
router,
mode,
linkedSources,
log,
configsByKey,
})

return processActionConfig({
garden,
graph,
config,
dependencies,
log,
configsByKey,
mode,
linkedSource,
remoteSourcePath,
templateContext,
supportedModes,
scanRoot,
})
Expand All @@ -346,23 +389,21 @@ async function processActionConfig({
graph,
config,
log,
configsByKey,
dependencies,
mode,
linkedSource,
remoteSourcePath,
templateContext,
supportedModes,
scanRoot,
}: {
garden: Garden
graph: ConfigGraph
config: ActionConfig
dependencies: ActionDependency[]
log: Log
configsByKey: ActionConfigsByKey
mode: ActionMode
linkedSource: LinkedSource | null
remoteSourcePath: string | null
templateContext: ActionConfigContext
supportedModes: ActionModes
scanRoot?: string
}) {
Expand Down Expand Up @@ -399,8 +440,6 @@ async function processActionConfig({
})
}

const dependencies = dependenciesFromActionConfig(log, config, configsByKey, definition, templateContext, actionTypes)

if (config.exclude?.includes("**/*")) {
if (config.include && config.include.length !== 0) {
throw new ConfigurationError({
Expand Down Expand Up @@ -602,10 +641,10 @@ function getActionSchema(kind: ActionKind) {

interface PreprocessActionResult {
config: ActionConfig
dependencies: ActionDependency[]
supportedModes: ActionModes
remoteSourcePath: string | null
linkedSource: LinkedSource | null
templateContext: ActionConfigContext
}

export const preprocessActionConfig = profileAsync(async function preprocessActionConfig({
Expand All @@ -615,13 +654,19 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi
mode,
linkedSources,
log,
configsByKey,
definition,
actionTypes,
}: {
garden: Garden
config: ActionConfig
router: ActionRouter
mode: ActionMode
linkedSources: LinkedSourceMap
log: Log
configsByKey: ActionConfigsByKey
definition: MaybeUndefined<ActionTypeDefinition<any>>
actionTypes: ActionDefinitionMap
}): Promise<PreprocessActionResult> {
const description = describeActionConfig(config)
const templateName = config.internal.templateName
Expand Down Expand Up @@ -823,23 +868,39 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi
}
}

const dependencies = dependenciesFromActionConfig({
log,
config,
configsByKey,
definition,
templateContext: builtinFieldContext,
actionTypes,
})

return {
config,
dependencies,
supportedModes,
remoteSourcePath,
linkedSource,
templateContext: builtinFieldContext,
}
})

function dependenciesFromActionConfig(
log: Log,
config: ActionConfig,
configsByKey: ActionConfigsByKey,
definition: MaybeUndefined<ActionTypeDefinition<any>>,
templateContext: ConfigContext,
function dependenciesFromActionConfig({
log,
config,
configsByKey,
definition,
templateContext,
actionTypes,
}: {
log: Log
config: ActionConfig
configsByKey: ActionConfigsByKey
definition: MaybeUndefined<ActionTypeDefinition<any>>
templateContext: ConfigContext
actionTypes: ActionDefinitionMap
) {
}) {
const description = describeActionConfig(config)

if (!config.dependencies) {
Expand Down
14 changes: 14 additions & 0 deletions core/test/unit/src/graph/actions.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,17 @@ describe("preprocessActionConfig", () => {
},
spec: { command: ["echo", "foo"] },
}

const router = await garden.getActionRouter()
const actionTypes = await garden.getActionTypes()
const definition = actionTypes[config.kind][config.type]?.spec

const res = await preprocessActionConfig({
garden,
config,
configsByKey: { "run.run": config },
actionTypes,
definition,
router,
linkedSources: {},
mode: "default",
Expand All @@ -69,10 +76,17 @@ describe("preprocessActionConfig", () => {
},
spec: { command: ["echo", "foo"] },
}

const router = await garden.getActionRouter()
const actionTypes = await garden.getActionTypes()
const definition = actionTypes[config.kind][config.type]?.spec

const res = await preprocessActionConfig({
garden,
config,
configsByKey: { "run.run": config },
actionTypes,
definition,
router,
linkedSources: {},
mode: "default",
Expand Down

0 comments on commit 2785f95

Please sign in to comment.