diff --git a/.gardenignore b/.gardenignore index 21d3a413c7..145bf7ef31 100644 --- a/.gardenignore +++ b/.gardenignore @@ -1,3 +1,7 @@ +.garden/ +.history/ +.vscode/ +.gradle/ node_modules/ tmp/ core/tmp/ diff --git a/.testignore b/.testignore new file mode 100644 index 0000000000..ca7941f6e4 --- /dev/null +++ b/.testignore @@ -0,0 +1,6 @@ +.garden/ +.history/ +.vscode/ +.gradle/ +tmp/ +core/tmp/ diff --git a/core/package.json b/core/package.json index 99bc23fbea..80cae08176 100644 --- a/core/package.json +++ b/core/package.json @@ -263,7 +263,8 @@ "integ-local": "GARDEN_INTEG_TEST_MODE=local GARDEN_SKIP_TESTS=\"remote-only\" yarn _integ", "integ-minikube": "GARDEN_INTEG_TEST_MODE=local GARDEN_SKIP_TESTS=\"remote-only\" yarn _integ", "integ-remote": "GARDEN_INTEG_TEST_MODE=remote GARDEN_SKIP_TESTS=local-only yarn _integ", - "test": "mocha" + "test": "mocha", + "test:silly": "GARDEN_LOGGER_TYPE=basic GARDEN_LOG_LEVEL=silly mocha" }, "pkg": { "scripts": [ diff --git a/core/src/commands/sync/sync-start.ts b/core/src/commands/sync/sync-start.ts index 12a63eb0b5..489fbfeaff 100644 --- a/core/src/commands/sync/sync-start.ts +++ b/core/src/commands/sync/sync-start.ts @@ -241,8 +241,8 @@ export async function startSyncWithoutDeploy({ someSyncStarted = true if (monitor) { - const monitor = new SyncMonitor({ garden, log, action: executedAction, graph, stopOnExit }) - garden.monitors.addAndSubscribe(monitor, command) + const m = new SyncMonitor({ garden, log, action: executedAction, graph, stopOnExit }) + garden.monitors.addAndSubscribe(m, command) } } catch (error) { actionLog.warn( diff --git a/core/src/config/project.ts b/core/src/config/project.ts index 919b9c40e0..4bc78a3832 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -30,7 +30,7 @@ import { findByName, getNames } from "../util/util" import { ConfigurationError, ParameterError, ValidationError } from "../exceptions" import { cloneDeep, memoize } from "lodash" import { GenericProviderConfig, providerConfigBaseSchema } from "./provider" -import { DOCS_BASE_URL, GardenApiVersion } from "../constants" +import { DOCS_BASE_URL, GardenApiVersion, GitScanMode, gitScanModes } from "../constants" import { defaultDotIgnoreFile } from "../util/fs" import type { CommandInfo } from "../plugin-context" import type { VcsInfo } from "../vcs/vcs" @@ -191,6 +191,10 @@ export interface ProxyConfig { hostname: string } +interface GitConfig { + mode: GitScanMode +} + export interface ProjectConfig { apiVersion: GardenApiVersion kind: "Project" @@ -207,6 +211,7 @@ export interface ProjectConfig { scan?: { include?: string[] exclude?: string[] + git?: GitConfig } outputs?: OutputSpec[] providers: GenericProviderConfig[] @@ -261,6 +266,16 @@ const projectScanSchema = createSchema({ ` ) .example(["public/**/*", "tmp/**/*"]), + git: joi.object().keys({ + mode: joi + .string() + .allow(...gitScanModes) + .only() + .default("subtree") + .description( + "Choose how to perform scans of git repositories. The default (`subtree`) runs individual git scans on each action/module path. The `repo` mode scans entire repositories and then filters down to files matching the paths, includes and excludes for each action/module. This can be considerably more efficient for large projects with many actions/modules." + ), + }), }), }) @@ -376,7 +391,7 @@ export const projectSchema = createSchema({ ) .example(["127.0.0.1"]), }), - scan: projectScanSchema().description("Control where to scan for configuration files in the project."), + scan: projectScanSchema().description("Control where and how to scan for configuration files in the project."), outputs: joiSparseArray(projectOutputSchema()) .unique("name") .description( diff --git a/core/src/constants.ts b/core/src/constants.ts index 0651e88abc..c5ef61c70f 100644 --- a/core/src/constants.ts +++ b/core/src/constants.ts @@ -12,6 +12,10 @@ import { homedir } from "os" export const isPkg = !!(process).pkg +export const gitScanModes = ["repo", "subtree"] as const +export type GitScanMode = (typeof gitScanModes)[number] +export const defaultGitScanMode: GitScanMode = "subtree" + export const GARDEN_CORE_ROOT = isPkg ? resolve(process.execPath, "..") : resolve(__dirname, "..", "..") export const GARDEN_CLI_ROOT = isPkg ? resolve(process.execPath, "..") : resolve(GARDEN_CORE_ROOT, "..", "cli") export const STATIC_DIR = isPkg ? resolve(process.execPath, "..", "static") : resolve(GARDEN_CORE_ROOT, "..", "static") @@ -62,6 +66,7 @@ export const gardenEnv = { GARDEN_ENVIRONMENT: env.get("GARDEN_ENVIRONMENT").required(false).asString(), GARDEN_EXPERIMENTAL_BUILD_STAGE: env.get("GARDEN_EXPERIMENTAL_BUILD_STAGE").required(false).asBool(), GARDEN_GE_SCHEDULED: env.get("GARDEN_GE_SCHEDULED").required(false).asBool(), + GARDEN_GIT_SCAN_MODE: env.get("GARDEN_GIT_SCAN_MODE").required(false).asEnum(gitScanModes), GARDEN_LEGACY_BUILD_STAGE: env.get("GARDEN_LEGACY_BUILD_STAGE").required(false).asBool(), GARDEN_LOG_LEVEL: env.get("GARDEN_LOG_LEVEL").required(false).asString(), GARDEN_LOGGER_TYPE: env.get("GARDEN_LOGGER_TYPE").required(false).asString(), diff --git a/core/src/garden.ts b/core/src/garden.ts index 248d489fa6..2e0db26810 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -152,6 +152,7 @@ import { getGardenInstanceKey } from "./server/helpers" import { SuggestedCommand } from "./commands/base" import { OtelTraced } from "./util/tracing/decorators" import { wrapActiveSpan } from "./util/tracing/spans" +import { GitRepoHandler } from "./vcs/git-repo" const defaultLocalAddress = "localhost" @@ -329,7 +330,10 @@ export class Garden { this.asyncLock = new AsyncLock() - this.vcs = new GitHandler({ + const gitMode = gardenEnv.GARDEN_GIT_SCAN_MODE || params.projectConfig.scan?.git?.mode + const handlerCls = gitMode === "repo" ? GitRepoHandler : GitHandler + + this.vcs = new handlerCls({ garden: this, projectRoot: params.projectRoot, gardenDirPath: params.gardenDirPath, @@ -1051,7 +1055,7 @@ export class Garden { let updated = false - // Resolve modules from specs and add to the list + // Resolve actions from augmentGraph specs and add to the list await Bluebird.map(addActions || [], async (config) => { // There is no actual config file for plugin modules (which the prepare function assumes) delete config.internal?.configFilePath @@ -1071,6 +1075,7 @@ export class Garden { configsByKey: actionConfigs, mode: actionModes[key] || "default", linkedSources, + scanRoot: config.internal.basePath, }) graph.addAction(action) @@ -1139,45 +1144,40 @@ export class Garden { @OtelTraced({ name: "resolveAction", }) - async resolveAction({ action, graph, log }: { action: T; log: Log; graph?: ConfigGraph }) { - if (!graph) { - graph = await this.getConfigGraph({ log, emit: false }) - } - + async resolveAction({ action, graph, log }: { action: T; log: Log; graph: ConfigGraph }) { return resolveAction({ garden: this, action, graph, log }) } @OtelTraced({ name: "resolveActions", }) - async resolveActions({ actions, graph, log }: { actions: T[]; log: Log; graph?: ConfigGraph }) { - if (!graph) { - graph = await this.getConfigGraph({ log, emit: false }) - } - + async resolveActions({ actions, graph, log }: { actions: T[]; log: Log; graph: ConfigGraph }) { return resolveActions({ garden: this, actions, graph, log }) } @OtelTraced({ name: "executeAction", }) - async executeAction({ action, graph, log }: { action: T; log: Log; graph?: ConfigGraph }) { - if (!graph) { - graph = await this.getConfigGraph({ log, emit: false }) - } - + async executeAction({ action, graph, log }: { action: T; log: Log; graph: ConfigGraph }) { return executeAction({ garden: this, action, graph, log }) } /** * Resolves the module version (i.e. build version) for the given module configuration and its build dependencies. */ - async resolveModuleVersion( - log: Log, - moduleConfig: ModuleConfig, - moduleDependencies: GardenModule[], - force = false - ): Promise { + async resolveModuleVersion({ + log, + moduleConfig, + moduleDependencies, + force = false, + scanRoot, + }: { + log: Log + moduleConfig: ModuleConfig + moduleDependencies: GardenModule[] + force?: boolean + scanRoot?: string + }): Promise { const moduleName = moduleConfig.name const depModuleNames = moduleDependencies.map((m) => m.name) depModuleNames.sort() @@ -1195,7 +1195,12 @@ export class Garden { const cacheContexts = [...moduleDependencies, moduleConfig].map((c: ModuleConfig) => getModuleCacheContext(c)) - const treeVersion = await this.vcs.getTreeVersion({ log, projectName: this.projectName, config: moduleConfig }) + const treeVersion = await this.vcs.getTreeVersion({ + log, + projectName: this.projectName, + config: moduleConfig, + scanRoot, + }) validateSchema(treeVersion, treeVersionSchema(), { context: `${this.vcs.name} tree version for module at ${moduleConfig.path}`, diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index f90b4e52b6..a3552c0695 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -64,6 +64,7 @@ import { LinkedSource, LinkedSourceMap } from "../config-store/local" import { relative } from "path" import { profileAsync } from "../util/profiling" import { uuidv4 } from "../util/random" +import { getConfigBasePath } from "../vcs/vcs" export const actionConfigsToGraph = profileAsync(async function actionConfigsToGraph({ garden, @@ -112,6 +113,10 @@ export const actionConfigsToGraph = profileAsync(async function actionConfigsToG } } + // Optimize file scanning by avoiding unnecessarily broad scans when project is not in repo root. + const allPaths = Object.values(configsByKey).map((c) => getConfigBasePath(c)) + const minimalRoots = await garden.vcs.getMinimalRoots(log, allPaths) + const router = await garden.getActionRouter() // TODO: Maybe we could optimize resolving tree versions, avoid parallel scanning of the same directory etc. @@ -151,7 +156,17 @@ export const actionConfigsToGraph = profileAsync(async function actionConfigsToG } try { - const action = await actionFromConfig({ garden, graph, config, router, log, configsByKey, mode, linkedSources }) + const action = await actionFromConfig({ + garden, + graph, + config, + router, + log, + configsByKey, + mode, + linkedSources, + scanRoot: minimalRoots[getConfigBasePath(config)], + }) if (!action.supportsMode(mode)) { if (explicitMode) { @@ -186,6 +201,7 @@ export const actionFromConfig = profileAsync(async function actionFromConfig({ configsByKey, mode, linkedSources, + scanRoot, }: { garden: Garden graph: ConfigGraph @@ -195,6 +211,7 @@ export const actionFromConfig = profileAsync(async function actionFromConfig({ configsByKey: ActionConfigsByKey mode: ActionMode linkedSources: LinkedSourceMap + scanRoot?: string }) { // Call configure handler and validate const { config, supportedModes, templateContext } = await preprocessActionConfig({ @@ -264,8 +281,10 @@ export const actionFromConfig = profileAsync(async function actionFromConfig({ } const dependencies = dependenciesFromActionConfig(log, config, configsByKey, definition, templateContext) + const treeVersion = - config.internal.treeVersion || (await garden.vcs.getTreeVersion({ log, projectName: garden.projectName, config })) + config.internal.treeVersion || + (await garden.vcs.getTreeVersion({ log, projectName: garden.projectName, config, scanRoot })) const variables = await mergeVariables({ basePath: config.internal.basePath, diff --git a/core/src/process.ts b/core/src/process.ts index d107e70f94..60f98ff3f3 100644 --- a/core/src/process.ts +++ b/core/src/process.ts @@ -112,5 +112,5 @@ export function isRunning(pid: number) { // Note: Circumvents an issue where the process exits before the output is fully flushed. // Needed for output renderers and Winston (see: https://github.com/winstonjs/winston/issues/228) export async function waitForOutputFlush() { - await sleep(100) + return sleep(50) } diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index c9948292db..8a461e936e 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -35,7 +35,7 @@ import type { ProviderMap } from "./config/provider" import chalk from "chalk" import { DependencyGraph } from "./graph/common" import Bluebird from "bluebird" -import { readFile, mkdirp, writeFile } from "fs-extra" +import { readFile, mkdirp } from "fs-extra" import type { Log } from "./logger/log-entry" import { ModuleConfigContext, ModuleConfigContextParams } from "./config/template-contexts/module" import { pathToCacheContext } from "./cache" @@ -100,14 +100,17 @@ export class ModuleResolver { // remove nodes from as we complete the processing. const fullGraph = new DependencyGraph() const processingGraph = new DependencyGraph() + const allPaths: string[] = [] for (const key of Object.keys(this.rawConfigsByKey)) { for (const graph of [fullGraph, processingGraph]) { graph.addNode(key) } } + for (const [key, rawConfig] of Object.entries(this.rawConfigsByKey)) { const buildPath = this.garden.buildStaging.getBuildPath(rawConfig) + allPaths.push(rawConfig.path) const deps = this.getModuleDependenciesFromConfig(rawConfig, buildPath) for (const graph of [fullGraph, processingGraph]) { for (const dep of deps) { @@ -118,6 +121,8 @@ export class ModuleResolver { } } + const minimalRoots = await this.garden.vcs.getMinimalRoots(this.log, allPaths) + const resolvedConfigs: ModuleConfigMap = {} const resolvedModules: ModuleMap = {} const errors: { [moduleName: string]: Error } = {} @@ -175,7 +180,12 @@ export class ModuleResolver { // dependencies. if (!foundNewDependency) { const buildPath = this.garden.buildStaging.getBuildPath(resolvedConfig) - resolvedModules[moduleKey] = await this.resolveModule(resolvedConfig, buildPath, resolvedDependencies) + resolvedModules[moduleKey] = await this.resolveModule({ + resolvedConfig, + buildPath, + dependencies: resolvedDependencies, + repoRoot: minimalRoots[resolvedConfig.path], + }) this.log.silly(`ModuleResolver: Module ${moduleKey} resolved`) processingGraph.removeNode(moduleKey) } @@ -478,7 +488,17 @@ export class ModuleResolver { return this.bases[type] } - private async resolveModule(resolvedConfig: ModuleConfig, buildPath: string, dependencies: GardenModule[]) { + private async resolveModule({ + resolvedConfig, + buildPath, + dependencies, + repoRoot, + }: { + resolvedConfig: ModuleConfig + buildPath: string + dependencies: GardenModule[] + repoRoot: string + }) { this.log.silly(`Resolving module ${resolvedConfig.name}`) // Write module files @@ -542,7 +562,8 @@ export class ModuleResolver { try { await mkdirp(targetDir) - await writeFile(targetPath, resolvedContents) + // Use VcsHandler.writeFile() to make sure version is re-computed after writing new/updated files + await this.garden.vcs.writeFile(this.log, targetPath, resolvedContents) } catch (error) { throw new FilesystemError({ message: `Unable to write templated file ${fileSpec.targetPath} from ${resolvedConfig.name}: ${error.message}`, @@ -565,6 +586,7 @@ export class ModuleResolver { log: this.log, config: resolvedConfig, buildDependencies: dependencies, + scanRoot: repoRoot, }) const moduleTypeDefinitions = await this.garden.getModuleTypes() @@ -745,7 +767,7 @@ export const convertModules = profileAsync(async function convertModules( const totalReturned = (result.actions?.length || 0) + (result.group?.actions.length || 0) - log.debug(`Module ${module.name} converted to ${totalReturned} actions`) + log.debug(`Module ${module.name} converted to ${totalReturned} action(s)`) if (result.group) { for (const action of result.group.actions) { diff --git a/core/src/types/module.ts b/core/src/types/module.ts index bc83bee051..c869a743aa 100644 --- a/core/src/types/module.ts +++ b/core/src/types/module.ts @@ -119,14 +119,22 @@ export async function moduleFromConfig({ config, buildDependencies, forceVersion = false, + scanRoot, }: { garden: Garden log: Log config: ModuleConfig buildDependencies: GardenModule[] forceVersion?: boolean + scanRoot?: string }): Promise { - const version = await garden.resolveModuleVersion(log, config, buildDependencies, forceVersion) + const version = await garden.resolveModuleVersion({ + log, + moduleConfig: config, + moduleDependencies: buildDependencies, + force: forceVersion, + scanRoot, + }) const actions = await garden.getActionRouter() const { outputs } = await actions.module.getModuleOutputs({ log, moduleConfig: config, version }) const moduleTypes = await garden.getModuleTypes() diff --git a/core/src/util/fs.ts b/core/src/util/fs.ts index 0603c6f43b..ab18491b27 100644 --- a/core/src/util/fs.ts +++ b/core/src/util/fs.ts @@ -176,15 +176,14 @@ export async function findConfigPathsInPath({ exclude = [] } - exclude.push(".garden/**/*") - const paths = await vcs.getFiles({ path: dir, pathDescription: "project root", include, - exclude: exclude || [], + exclude, log, filter: (f) => isConfigFilename(basename(f)), + scanRoot: dir, }) return paths.map((f) => f.path) @@ -260,13 +259,14 @@ export function matchGlobs(paths: string[], patterns: string[]): string[] { /** * Check if a path passes through given include/exclude filters. * - * @param path A POSIX-style path + * @param path A filesystem path * @param include List of globs to match for inclusion, or undefined * @param exclude List of globs to match for exclusion, or undefined */ export function matchPath(path: string, include?: string[], exclude?: string[]) { return ( - (!include || matchGlobs([path], include).length === 1) && (!exclude || matchGlobs([path], exclude).length === 0) + (!include || matchGlobs([path], include).length === 1) && + (!exclude?.length || matchGlobs([path], exclude).length === 0) ) } diff --git a/core/src/util/testing.ts b/core/src/util/testing.ts index 1abd17cdd9..0b4f1455b2 100644 --- a/core/src/util/testing.ts +++ b/core/src/util/testing.ts @@ -142,7 +142,7 @@ export class TestEventBus extends EventBus { } const defaultCommandinfo = { name: "test", args: {}, opts: {} } -const repoRoot = resolve(GARDEN_CORE_ROOT, "..") +export const repoRoot = resolve(GARDEN_CORE_ROOT, "..") const paramCache: { [key: string]: GardenParams } = {} // const configGraphCache: { [key: string]: ConfigGraph } = {} diff --git a/core/src/vcs/git-repo.ts b/core/src/vcs/git-repo.ts new file mode 100644 index 0000000000..538dda7f01 --- /dev/null +++ b/core/src/vcs/git-repo.ts @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { join } from "path" +import { GitHandler, augmentGlobs } from "./git" +import type { GetFilesParams, VcsFile } from "./vcs" +import { isDirectory, joinWithPosix, matchPath } from "../util/fs" +import { pathExists } from "fs-extra" +import { pathToCacheContext } from "../cache" + +type ScanRepoParams = Pick + +export class GitRepoHandler extends GitHandler { + name = "git-repo" + + /** + * This has the same signature as the GitHandler super class method but instead of scanning the individual directory + * path directly, we scan the entire enclosing git repository, cache that file list and then filter down to the + * sub-path. This results in far fewer git process calls but in turn collects more data in memory. + */ + async getFiles(params: GetFilesParams): Promise { + const { log, path, pathDescription, filter, failOnPrompt = false } = params + + if (params.include && params.include.length === 0) { + // No need to proceed, nothing should be included + return [] + } + + if (!(await pathExists(path))) { + log.warn(`${pathDescription} ${path} could not be found.`) + return [] + } + + if (!(await isDirectory(path))) { + log.warn(`Path ${path} is not a directory.`) + return [] + } + + let scanRoot = params.scanRoot || path + + if (!params.scanRoot && params.pathDescription !== "submodule") { + scanRoot = await this.getRepoRoot(log, path, failOnPrompt) + } + + const repoFiles = await this.scanRepo({ + log, + path: scanRoot, + pathDescription: pathDescription || "repository", + failOnPrompt, + }) + + const include = params.include ? await absGlobs(path, params.include) : [path, join(path, "**", "*")] + const exclude = await absGlobs(path, params.exclude || []) + + if (scanRoot === this.garden?.projectRoot) { + exclude.push(join(scanRoot, ".garden", "**", "*")) + } + + return repoFiles.filter(({ path: p }) => (!filter || filter(p)) && matchPath(p, include, exclude)) + } + + /** + * Scans the given repo root and caches the list of files in the tree cache. + * Uses an async lock to ensure a repo root is only scanned once. + */ + async scanRepo(params: ScanRepoParams) { + const { log, path } = params + + const key = ["git-repo-files", path] + let existing = this.cache.get(log, key) + + if (existing) { + return existing + } + + return this.lock.acquire(key.join("|"), async () => { + existing = this.cache.get(log, key) + + if (existing) { + return existing + } + + params.log.info(`Scanning repository at ${path}`) + const files = await super.getFiles({ ...params, scanRoot: undefined }) + + this.cache.set(log, key, files, pathToCacheContext(path)) + + return files + }) + } +} + +async function absGlobs(basePath: string, globs: string[]): Promise { + const augmented = await augmentGlobs(basePath, globs) + return augmented?.map((p) => joinWithPosix(basePath, p)) || [] +} diff --git a/core/src/vcs/git.ts b/core/src/vcs/git.ts index 555473c842..1f2032f04b 100644 --- a/core/src/vcs/git.ts +++ b/core/src/vcs/git.ts @@ -16,11 +16,10 @@ import { ConfigurationError, RuntimeError } from "../exceptions" import Bluebird from "bluebird" import { getStatsType, joinWithPosix, matchPath } from "../util/fs" import { dedent, deline, splitLast } from "../util/string" -import { exec } from "../util/util" +import { exec, sleep } from "../util/util" import { Log } from "../logger/log-entry" import parseGitConfig from "parse-git-config" import { getDefaultProfiler, Profile, Profiler } from "../util/profiling" -import { mapLimit } from "async" import { STATIC_DIR } from "../constants" import split2 = require("split2") import execa = require("execa") @@ -33,7 +32,6 @@ import AsyncLock from "async-lock" const gitConfigAsyncLock = new AsyncLock() const submoduleErrorSuggestion = `Perhaps you need to run ${chalk.underline(`git submodule update --recursive`)}?` -const hashConcurrencyLimit = 50 const currentPlatformName = process.platform const gitSafeDirs = new Set() @@ -62,7 +60,7 @@ export function parseGitUrl(url: string) { return parsed } -interface GitCli { +export interface GitCli { (...args: (string | undefined)[]): Promise } @@ -82,7 +80,7 @@ export class GitHandler extends VcsHandler { name = "git" repoRoots = new Map() profiler: Profiler - private lock: AsyncLock + protected lock: AsyncLock constructor(params: VcsHandlerParams) { super(params) @@ -237,7 +235,7 @@ export class GitHandler extends VcsHandler { }) } else if (err.exitCode === 128) { // Throw nice error when we detect that we're not in a repo root - throw new RuntimeError({ message: notInRepoRootErrorMessage(path), detail: { path } }) + throw new RuntimeError({ message: notInRepoRootErrorMessage(path), detail: { path, exitCode: err.exitCode } }) } else { throw err } @@ -245,11 +243,7 @@ export class GitHandler extends VcsHandler { }) } - /** - * Returns a list of files, along with file hashes, under the given path, taking into account the configured - * .ignore files, and the specified include/exclude filters. - */ - async getFiles({ + async streamPaths({ log, path, pathDescription = "directory", @@ -257,16 +251,24 @@ export class GitHandler extends VcsHandler { exclude, filter, failOnPrompt = false, - }: GetFilesParams): Promise { + callback, + }: GetFilesParams & { callback: (err: Error | null, entry?: VcsFile) => void }): Promise { if (include && include.length === 0) { // No need to proceed, nothing should be included - return [] + return callback(null) + } + + if (!exclude) { + exclude = [] } + exclude.push("**/.garden/**/*") const gitLog = log .createLog({}) .debug( - `Scanning ${pathDescription} at ${path}\nā†’ Includes: ${include || "(none)"}\nā†’ Excludes: ${exclude || "(none)"}` + `Scanning ${pathDescription} at ${path}\n ā†’ Includes: ${include || "(none)"}\n ā†’ Excludes: ${ + exclude || "(none)" + }` ) try { @@ -274,13 +276,13 @@ export class GitHandler extends VcsHandler { if (!pathStats.isDirectory()) { gitLog.warn(`Expected directory at ${path}, but found ${getStatsType(pathStats)}.`) - return [] + return callback(null) } } catch (err) { // 128 = File no longer exists if (err.exitCode === 128 || err.code === "ENOENT") { gitLog.warn(`Attempted to scan directory at ${path}, but it does not exist.`) - return [] + return callback(null) } else { throw err } @@ -296,7 +298,7 @@ export class GitHandler extends VcsHandler { .map((modifiedRelPath) => resolve(gitRoot, modifiedRelPath)) ) - const absExcludes = exclude ? exclude.map((p) => resolve(path, p)) : undefined + const absExcludes = exclude.map((p) => resolve(path, p)) // Apply the include patterns to the ls-files queries. We use the 'glob' "magic word" (in git parlance) // to make sure the path handling is consistent with normal POSIX-style globs used generally by Garden. @@ -318,33 +320,32 @@ export class GitHandler extends VcsHandler { gitLog.silly(`Submodules listed at ${submodules.map((s) => `${s.path} (${s.url})`).join(", ")}`) } - const files: VcsFile[] = [] - - const parseLine = (data: Buffer): VcsFile | undefined => { - const line = data.toString().trim() - if (!line) { - return undefined - } - - let filePath: string - let hash = "" - - const split = line.trim().split("\t") + // Make sure we have a fresh hash for each file + let count = 0 - if (split.length === 1) { - // File is untracked - filePath = split[0] - } else { - filePath = split[1] - hash = split[0].split(" ")[1] + const ensureHash = (entry: FileEntry, stats: Stats, reject: (err: Error) => void) => { + if (entry.hash === "" || modified.has(entry.path)) { + // Don't attempt to hash directories. Directories will by extension be filtered out of the list. + if (!stats.isDirectory()) { + return this.hashObject(stats, entry.path, (err, hash) => { + if (err) { + return reject(err) + } + if (hash !== "") { + entry.hash = hash + count++ + return callback(null, entry) + } + }) + } } - - return { path: filePath, hash } + count++ + callback(null, entry) } // This function is called for each line output from the ls-files commands that we run, and populates the // `files` array. - const handleEntry = (entry: VcsFile | undefined) => { + const handleEntry = (entry: VcsFile | undefined, reject: (err: Error) => void) => { if (!entry) { return } @@ -355,7 +356,6 @@ export class GitHandler extends VcsHandler { if (filter && !filter(filePath)) { return } - // Ignore files that are tracked but still specified in ignore files if (trackedButIgnored.has(filePath)) { return @@ -363,36 +363,91 @@ export class GitHandler extends VcsHandler { const resolvedPath = resolve(path, filePath) - // We push to the output array if it passes through the exclude filters. - if (matchPath(filePath, undefined, exclude) && !submodulePaths.includes(resolvedPath)) { - files.push({ path: resolvedPath, hash }) + // Filter on excludes + if (!matchPath(filePath, undefined, exclude) || submodulePaths.includes(resolvedPath)) { + return } + + // We push to the output array if it passes through the exclude filters. + const output = { path: resolvedPath, hash: hash || "" } + + return lstat(resolvedPath, (err, stats) => { + if (err) { + if (err.code === "ENOENT") { + return + } + return reject(err) + } + + // We need to special-case handling of symlinks. We disallow any "unsafe" symlinks, i.e. any ones that may + // link outside of `gitRoot`. + if (stats.isSymbolicLink()) { + return readlink(resolvedPath, (readlinkErr, target) => { + if (readlinkErr) { + return reject(readlinkErr) + } + + // Make sure symlink is relative and points within `path` + if (isAbsolute(target)) { + gitLog.verbose(`Ignoring symlink with absolute target at ${resolvedPath}`) + } else if (target.startsWith("..")) { + return realpath(resolvedPath, (realpathErr, realTarget) => { + if (realpathErr) { + if (realpathErr.code === "ENOENT") { + gitLog.verbose(`Ignoring dead symlink at ${resolvedPath}`) + return + } + return reject(realpathErr) + } + + const relPath = relative(path, realTarget) + + if (relPath.startsWith("..")) { + gitLog.verbose(`Ignoring symlink pointing outside of ${pathDescription} at ${resolvedPath}`) + return + } + return ensureHash(output, stats, reject) + }) + } else { + return ensureHash(output, stats, reject) + } + }) + } + return ensureHash(output, stats, reject) + }) } - const lsFiles = (ignoreFile?: string) => { + await new Promise((_resolve, _reject) => { + // Prepare args const args = ["ls-files", "-s", "--others", ...lsFilesCommonArgs] - - if (ignoreFile) { - args.push("--exclude-per-directory", ignoreFile) + if (this.ignoreFile) { + args.push("--exclude-per-directory", this.ignoreFile) } args.push(...patterns) + // Start git process gitLog.silly(`Calling git with args '${args.join(" ")}' in ${path}`) - return execa("git", args, { cwd: path, buffer: false }) - } + const proc = execa("git", args, { cwd: path, buffer: false }) - const splitStream = split2() - splitStream.on("data", (line) => handleEntry(parseLine(line))) + // Stream + const fail = (err: Error) => { + _reject(err) + proc.kill() + splitStream.end() + } + const splitStream = split2() + splitStream.on("data", (line) => { + handleEntry(parseLine(line), fail) + }) - await new Promise((_resolve, _reject) => { - const proc = lsFiles(this.ignoreFile) void proc.on("error", (err: execa.ExecaError) => { if (err.exitCode !== 128) { - _reject(err) + fail(err) } }) proc.stdout?.pipe(splitStream) - splitStream.on("end", () => _resolve()) + // The sleep here is necessary to wrap up callbacks + splitStream.on("end", () => sleep(30).then(() => _resolve())) }) if (submodulePaths.length > 0) { @@ -403,120 +458,68 @@ export class GitHandler extends VcsHandler { // Resolve submodules // TODO: see about optimizing this, avoiding scans when we're sure they'll not match includes/excludes etc. await Bluebird.map(submodulePaths, async (submodulePath) => { - if (submodulePath.startsWith(path) && !absExcludes?.includes(submodulePath)) { - // Note: We apply include/exclude filters after listing files from submodule - const submoduleRelPath = relative(path, submodulePath) - - // Catch and show helpful message in case the submodule path isn't a valid directory - try { - const pathStats = await stat(path) - - if (!pathStats.isDirectory()) { - const pathType = getStatsType(pathStats) - gitLog.warn(`Expected submodule directory at ${path}, but found ${pathType}. ${submoduleErrorSuggestion}`) - return - } - } catch (err) { - // 128 = File no longer exists - if (err.exitCode === 128 || err.code === "ENOENT") { - gitLog.warn( - `Found reference to submodule at ${submoduleRelPath}, but the path could not be found. ${submoduleErrorSuggestion}` - ) - return - } else { - throw err - } - } - - files.push( - ...(await this.getFiles({ - log: gitLog, - path: submodulePath, - pathDescription: "submodule", - exclude: [], - filter: (p) => matchPath(join(submoduleRelPath, p), augmentedIncludes, augmentedExcludes), - })) - ) - } - }) - } - - // Make sure we have a fresh hash for each file - const _this = this - - function ensureHash(entry: FileEntry, stats: Stats, cb: (err: Error | null, entry?: FileEntry) => void) { - if (entry.hash === "" || modified.has(entry.path)) { - // Don't attempt to hash directories. Directories will by extension be filtered out of the list. - if (!stats.isDirectory()) { - return _this.hashObject(stats, entry.path, (err, hash) => { - if (err) { - return cb(err) - } - entry.hash = hash || "" - cb(null, entry) - }) + if (!submodulePath.startsWith(path) || absExcludes?.includes(submodulePath)) { + return } - } - cb(null, entry) - } + // Note: We apply include/exclude filters after listing files from submodule + const submoduleRelPath = relative(path, submodulePath) - const result = ( - await mapLimit(files, hashConcurrencyLimit, (f, cb) => { - const resolvedPath = resolve(path, f.path) - const output = { path: resolvedPath, hash: f.hash || "" } + // Catch and show helpful message in case the submodule path isn't a valid directory + try { + const pathStats = await stat(path) - lstat(resolvedPath, (err, stats) => { - if (err) { - if (err.code === "ENOENT") { - return cb(null, { path: resolvedPath, hash: "" }) - } - return cb(err) + if (!pathStats.isDirectory()) { + const pathType = getStatsType(pathStats) + gitLog.warn(`Expected submodule directory at ${path}, but found ${pathType}. ${submoduleErrorSuggestion}`) + return } - - // We need to special-case handling of symlinks. We disallow any "unsafe" symlinks, i.e. any ones that may - // link outside of `gitRoot`. - if (stats.isSymbolicLink()) { - readlink(resolvedPath, (readlinkErr, target) => { - if (readlinkErr) { - return cb(readlinkErr) - } - - // Make sure symlink is relative and points within `path` - if (isAbsolute(target)) { - gitLog.verbose(`Ignoring symlink with absolute target at ${resolvedPath}`) - return cb(null, { path: resolvedPath, hash: "" }) - } else if (target.startsWith("..")) { - realpath(resolvedPath, (realpathErr, realTarget) => { - if (realpathErr) { - if (realpathErr.code === "ENOENT") { - return cb(null, { path: resolvedPath, hash: "" }) - } - return cb(err) - } - - const relPath = relative(path, realTarget) - - if (relPath.startsWith("..")) { - gitLog.verbose(`Ignoring symlink pointing outside of ${pathDescription} at ${resolvedPath}`) - return cb(null, { path: resolvedPath, hash: "" }) - } - ensureHash(output, stats, cb) - }) - } else { - ensureHash(output, stats, cb) - } - }) + } catch (err) { + // 128 = File no longer exists + if (err.exitCode === 128 || err.code === "ENOENT") { + gitLog.warn( + `Found reference to submodule at ${submoduleRelPath}, but the path could not be found. ${submoduleErrorSuggestion}` + ) + return } else { - ensureHash(output, stats, cb) + throw err } + } + + return this.streamPaths({ + log: gitLog, + path: submodulePath, + pathDescription: "submodule", + exclude: [], + filter: (p) => + matchPath(join(submoduleRelPath, p), augmentedIncludes, augmentedExcludes) && (!filter || filter(p)), + scanRoot: submodulePath, + failOnPrompt, + callback, }) }) - ).filter((f) => f.hash !== "") + } + + gitLog.debug(`Found ${count} files in ${pathDescription} ${path}`) + } + + /** + * Returns a list of files, along with file hashes, under the given path, taking into account the configured + * .ignore files, and the specified include/exclude filters. + */ + async getFiles(params: GetFilesParams): Promise { + const files: VcsFile[] = [] - gitLog.debug(`Found ${result.length} files in ${pathDescription} ${path}`) + await this.streamPaths({ + ...params, + callback: (_, entry) => { + if (entry) { + files.push(entry) + } + }, + }) - return result + return files.sort() } private isHashSHA1(hash: string): boolean { @@ -751,7 +754,7 @@ const notInRepoRootErrorMessage = (path: string) => deline` * Given a list of POSIX-style globs/paths and a `basePath`, find paths that point to a directory and append `**\/*` * to them, such that they'll be matched consistently between git and our internal pattern matching. */ -async function augmentGlobs(basePath: string, globs?: string[]) { +export async function augmentGlobs(basePath: string, globs?: string[]) { if (!globs) { return globs } @@ -764,9 +767,31 @@ async function augmentGlobs(basePath: string, globs?: string[]) { try { const isDir = (await stat(joinWithPosix(basePath, pattern))).isDirectory() - return isDir ? posix.join(pattern, "**/*") : pattern + return isDir ? posix.join(pattern, "**", "*") : pattern } catch { return pattern } }) } + +const parseLine = (data: Buffer): VcsFile | undefined => { + const line = data.toString().trim() + if (!line) { + return undefined + } + + let filePath: string + let hash = "" + + const split = line.trim().split("\t") + + if (split.length === 1) { + // File is untracked + filePath = split[0] + } else { + filePath = split[1] + hash = split[0].split(" ")[1] + } + + return { path: filePath, hash } +} diff --git a/core/src/vcs/vcs.ts b/core/src/vcs/vcs.ts index f1152c6844..3427d1dd6e 100644 --- a/core/src/vcs/vcs.ts +++ b/core/src/vcs/vcs.ts @@ -7,11 +7,11 @@ */ import Joi from "@hapi/joi" -import normalize = require("normalize-path") +import normalize from "normalize-path" import { sortBy, pick } from "lodash" import { createHash } from "crypto" import { validateSchema } from "../config/validation" -import { join, relative, isAbsolute } from "path" +import { join, relative, isAbsolute, sep } from "path" import { DOCS_BASE_URL, GARDEN_VERSIONFILE_NAME as GARDEN_TREEVERSION_FILENAME } from "../constants" import { pathExists, readFile, writeFile } from "fs-extra" import { ConfigurationError } from "../exceptions" @@ -19,7 +19,7 @@ import { ExternalSourceType, getRemoteSourceLocalPath, getRemoteSourcesPath } fr import { ModuleConfig, serializeConfig } from "../config/module" import type { Log } from "../logger/log-entry" import { treeVersionSchema } from "../config/common" -import { dedent } from "../util/string" +import { dedent, splitLast } from "../util/string" import { fixedProjectExcludes } from "../util/fs" import { pathToCacheContext, TreeCache } from "../cache" import type { ServiceConfig } from "../config/service" @@ -30,8 +30,9 @@ import { validateInstall } from "../util/validateInstall" import { isActionConfig } from "../actions/base" import type { BaseActionConfig } from "../actions/types" import { Garden } from "../garden" -import chalk = require("chalk") +import chalk from "chalk" import { Profile } from "../util/profiling" +import Bluebird from "bluebird" const AsyncLock = require("async-lock") const scanLock = new AsyncLock() @@ -104,6 +105,14 @@ export interface GetFilesParams { exclude?: string[] filter?: (path: string) => boolean failOnPrompt?: boolean + scanRoot: string | undefined +} + +export interface GetTreeVersionParams { + log: Log + projectName: string + config: ModuleConfig | BaseActionConfig + scanRoot?: string // Set the scanning root instead of detecting, in order to optimize the scanning. } export interface RemoteSourceParams { @@ -133,7 +142,7 @@ export abstract class VcsHandler { protected projectRoot: string protected gardenDirPath: string protected ignoreFile: string - private cache: TreeCache + protected cache: TreeCache constructor(params: VcsHandlerParams) { this.garden = params.garden @@ -146,13 +155,9 @@ export abstract class VcsHandler { abstract name: string abstract getRepoRoot(log: Log, path: string): Promise - abstract getFiles(params: GetFilesParams): Promise - abstract ensureRemoteSource(params: RemoteSourceParams): Promise - abstract updateRemoteSource(params: RemoteSourceParams): Promise - abstract getPathInfo(log: Log, path: string): Promise clearTreeCache() { @@ -164,11 +169,13 @@ export abstract class VcsHandler { projectName, config, force = false, + scanRoot, }: { log: Log projectName: string config: ModuleConfig | BaseActionConfig force?: boolean + scanRoot?: string }): Promise { const cacheKey = getResourceTreeCacheKey(config) const description = describeConfig(config) @@ -212,6 +219,7 @@ export abstract class VcsHandler { pathDescription: description + " root", include: config.include, exclude, + scanRoot, }) if (files.length > fileCountWarningThreshold) { @@ -241,11 +249,75 @@ export abstract class VcsHandler { return result } - async resolveTreeVersion(log: Log, projectName: string, moduleConfig: ModuleConfig): Promise { + /** + * Write a file and ensure relevant caches are invalidated after writing. + */ + async writeFile(log: Log, path: string, data: string | Buffer) { + await writeFile(path, data) + this.cache.invalidateUp(log, pathToCacheContext(path)) + } + + async resolveTreeVersion(params: GetTreeVersionParams): Promise { // the version file is used internally to specify versions outside of source control - const versionFilePath = join(moduleConfig.path, GARDEN_TREEVERSION_FILENAME) + const path = getConfigBasePath(params.config) + const versionFilePath = join(path, GARDEN_TREEVERSION_FILENAME) const fileVersion = await readTreeVersionFile(versionFilePath) - return fileVersion || (await this.getTreeVersion({ log, projectName, config: moduleConfig })) + return fileVersion || (await this.getTreeVersion(params)) + } + + /** + * Returns a map of the optimal paths for each of the given action/module source path. + * This is used to avoid scanning more of each git repository than necessary, and + * reduces duplicate scanning of the same directories (since fewer unique roots mean + * more tree cache hits). + */ + async getMinimalRoots(log: Log, paths: string[]) { + const repoRoots: { [path: string]: string } = {} + const outputs: { [path: string]: string } = {} + const rootsToPaths: { [repoRoot: string]: string[] } = {} + + await Bluebird.map(paths, async (path) => { + const repoRoot = await this.getRepoRoot(log, path) + repoRoots[path] = repoRoot + if (rootsToPaths[repoRoot]) { + rootsToPaths[repoRoot].push(path) + } else { + rootsToPaths[repoRoot] = [path] + } + }) + + for (const path of paths) { + const repoRoot = repoRoots[path] + const repoPaths = rootsToPaths[repoRoot] + + for (const repoPath of repoPaths) { + if (!outputs[path]) { + // No path set so far + outputs[path] = repoPath + } else if (outputs[path].startsWith(repoPath)) { + // New path is prefix of prior path + outputs[path] = repoPath + } else { + // Find common prefix + let p = repoPath + + while (true) { + p = splitLast(p, sep)[0] + if (p.length < repoRoot.length) { + // Don't go past the actual git repo root + outputs[path] = repoRoot + break + } else if (outputs[path].startsWith(p)) { + // Found a common prefix + outputs[path] = p + break + } + } + } + } + } + + return outputs } /** diff --git a/core/test/integ/src/plugins/container/container.ts b/core/test/integ/src/plugins/container/container.ts index 8c5976fc10..df3922992d 100644 --- a/core/test/integ/src/plugins/container/container.ts +++ b/core/test/integ/src/plugins/container/container.ts @@ -215,7 +215,11 @@ describe("plugins.container", () => { config.spec.extraFlags = ["--cache-from", "some-image:latest"] const buildAction = await getTestBuild(config) - const resolvedBuild = await garden.resolveAction({ action: buildAction, log }) + const resolvedBuild = await garden.resolveAction({ + action: buildAction, + log, + graph: await garden.getConfigGraph({ log, emit: false }), + }) const args = getDockerBuildFlags(resolvedBuild) @@ -227,7 +231,11 @@ describe("plugins.container", () => { const buildAction = await getTestBuild(config) - const resolvedBuild = await garden.resolveAction({ action: buildAction, log }) + const resolvedBuild = await garden.resolveAction({ + action: buildAction, + log, + graph: await garden.getConfigGraph({ log, emit: false }), + }) const args = getDockerBuildFlags(resolvedBuild) diff --git a/core/test/integ/src/plugins/kubernetes/util.ts b/core/test/integ/src/plugins/kubernetes/util.ts index 54a2fefe1e..8d3f66dd61 100644 --- a/core/test/integ/src/plugins/kubernetes/util.ts +++ b/core/test/integ/src/plugins/kubernetes/util.ts @@ -232,8 +232,12 @@ describe("util", () => { describe("getTargetResource", () => { it("should return the resource specified by the query", async () => { const rawAction = helmGraph.getDeploy("api") - const action = await helmGarden.resolveAction({ action: rawAction, log: helmGarden.log }) - await helmGarden.executeAction({ action: rawAction, log: helmGarden.log }) + const action = await helmGarden.resolveAction({ + action: rawAction, + log: helmGarden.log, + graph: helmGraph, + }) + await helmGarden.executeAction({ action: rawAction, log: helmGarden.log, graph: helmGraph }) const manifests = await getChartResources({ ctx, action, @@ -257,7 +261,11 @@ describe("util", () => { it("should throw if no query is specified", async () => { const rawAction = helmGraph.getDeploy("api") - const action = await helmGarden.resolveAction({ action: rawAction, log: helmGarden.log }) + const action = await helmGarden.resolveAction({ + action: rawAction, + log: helmGarden.log, + graph: helmGraph, + }) const manifests = await getChartResources({ ctx, action, @@ -281,7 +289,11 @@ describe("util", () => { it("should throw if no resource of the specified kind is in the chart", async () => { const rawAction = helmGraph.getDeploy("api") - const action = await helmGarden.resolveAction({ action: rawAction, log: helmGarden.log }) + const action = await helmGarden.resolveAction({ + action: rawAction, + log: helmGarden.log, + graph: helmGraph, + }) const manifests = await getChartResources({ ctx, action, @@ -307,7 +319,11 @@ describe("util", () => { it("should throw if matching resource is not found by name", async () => { const rawAction = helmGraph.getDeploy("api") - const action = await helmGarden.resolveAction({ action: rawAction, log: helmGarden.log }) + const action = await helmGarden.resolveAction({ + action: rawAction, + log: helmGarden.log, + graph: helmGraph, + }) const manifests = await getChartResources({ ctx, action, @@ -333,7 +349,11 @@ describe("util", () => { it("should throw if no name is specified and multiple resources are matched", async () => { const rawAction = helmGraph.getDeploy("api") - const action = await helmGarden.resolveAction({ action: rawAction, log: helmGarden.log }) + const action = await helmGarden.resolveAction({ + action: rawAction, + log: helmGarden.log, + graph: helmGraph, + }) const manifests = await getChartResources({ ctx, action, @@ -364,7 +384,11 @@ describe("util", () => { it("should resolve template string for resource name", async () => { const rawAction = helmGraph.getDeploy("postgres") - const action = await helmGarden.resolveAction({ action: rawAction, log: helmGarden.log }) + const action = await helmGarden.resolveAction({ + action: rawAction, + log: helmGarden.log, + graph: helmGraph, + }) const manifests = await getChartResources({ ctx, action, @@ -389,7 +413,11 @@ describe("util", () => { context("podSelector", () => { before(async () => { const rawAction = helmGraph.getDeploy("api") - const action = await helmGarden.resolveAction({ action: rawAction, log: helmGarden.log }) + const action = await helmGarden.resolveAction({ + action: rawAction, + log: helmGarden.log, + graph: helmGraph, + }) const deployTask = new DeployTask({ force: false, forceBuild: false, @@ -404,7 +432,11 @@ describe("util", () => { it("returns running Pod if one is found matching podSelector", async () => { const rawAction = helmGraph.getDeploy("api") - const action = await helmGarden.resolveAction({ action: rawAction, log: helmGarden.log }) + const action = await helmGarden.resolveAction({ + action: rawAction, + log: helmGarden.log, + graph: helmGraph, + }) const resourceSpec: ServiceResourceSpec = { podSelector: { "app.kubernetes.io/name": "api", @@ -429,7 +461,11 @@ describe("util", () => { it("throws if podSelector is set and no Pod is found matching the selector", async () => { const rawAction = helmGraph.getDeploy("api") - const action = await helmGarden.resolveAction({ action: rawAction, log: helmGarden.log }) + const action = await helmGarden.resolveAction({ + action: rawAction, + log: helmGarden.log, + graph: helmGraph, + }) const resourceSpec: ServiceResourceSpec = { podSelector: { "app.kubernetes.io/name": "boo", @@ -502,7 +538,11 @@ describe("util", () => { describe("getResourceContainer", () => { async function getK8sDeployment() { const rawAction = helmGraph.getDeploy("api") - const action = await helmGarden.resolveAction({ action: rawAction, log: helmGarden.log }) + const action = await helmGarden.resolveAction({ + action: rawAction, + log: helmGarden.log, + graph: helmGraph, + }) const manifests = await getChartResources({ ctx, action, diff --git a/core/test/integ/src/plugins/kubernetes/volume/configmap.ts b/core/test/integ/src/plugins/kubernetes/volume/configmap.ts index bd9e49c898..b419d2fc63 100644 --- a/core/test/integ/src/plugins/kubernetes/volume/configmap.ts +++ b/core/test/integ/src/plugins/kubernetes/volume/configmap.ts @@ -64,7 +64,7 @@ describe("configmap module", () => { ]) const graph = await garden.getConfigGraph({ log: garden.log, emit: false }) - const action = await garden.resolveAction({ action: graph.getDeploy("test"), log: garden.log }) + const action = await garden.resolveAction({ action: graph.getDeploy("test"), log: garden.log, graph }) const deployTask = new DeployTask({ garden, diff --git a/core/test/integ/src/plugins/kubernetes/volume/persistentvolumeclaim.ts b/core/test/integ/src/plugins/kubernetes/volume/persistentvolumeclaim.ts index d3e75893ee..f4d3c3010c 100644 --- a/core/test/integ/src/plugins/kubernetes/volume/persistentvolumeclaim.ts +++ b/core/test/integ/src/plugins/kubernetes/volume/persistentvolumeclaim.ts @@ -69,7 +69,7 @@ describe("persistentvolumeclaim", () => { ]) const graph = await garden.getConfigGraph({ log: garden.log, emit: false }) - const action = await garden.resolveAction({ action: graph.getDeploy("test"), log: garden.log }) + const action = await garden.resolveAction({ action: graph.getDeploy("test"), log: garden.log, graph }) const deployTask = new DeployTask({ garden, diff --git a/core/test/unit/src/build-staging/build-staging.ts b/core/test/unit/src/build-staging/build-staging.ts index de72956e9e..34a6c37ae7 100644 --- a/core/test/unit/src/build-staging/build-staging.ts +++ b/core/test/unit/src/build-staging/build-staging.ts @@ -358,8 +358,8 @@ function commonSyncTests(legacyBuildSync: boolean) { await garden.processTasks({ tasks: buildTasks }) - const buildActionD = await graph.getBuild("module-d") - const buildActionF = await graph.getBuild("module-f") + const buildActionD = graph.getBuild("module-d") + const buildActionF = graph.getBuild("module-f") const buildDirD = buildStaging.getBuildPath(buildActionD.getConfig()) const buildDirF = buildStaging.getBuildPath(buildActionF.getConfig()) @@ -426,7 +426,7 @@ function commonSyncTests(legacyBuildSync: boolean) { it("should ensure that a module's build subdir exists before returning from buildPath", async () => { const graph = await garden.getConfigGraph({ log: garden.log, emit: false }) - const buildActionA = await graph.getBuild("module-a") + const buildActionA = graph.getBuild("module-a") const buildPath = await buildStaging.ensureBuildPath(buildActionA.getConfig()) expect(await pathExists(buildPath)).to.eql(true) }) diff --git a/core/test/unit/src/commands/run.ts b/core/test/unit/src/commands/run.ts index c5cedab357..2fb1f46998 100644 --- a/core/test/unit/src/commands/run.ts +++ b/core/test/unit/src/commands/run.ts @@ -127,7 +127,7 @@ describe("RunCommand", () => { name: "task-disabled", disabled: true, internal: { - basePath: "/foo", + basePath: garden.projectRoot, }, timeout: DEFAULT_RUN_TIMEOUT_SEC, spec: { @@ -160,7 +160,7 @@ describe("RunCommand", () => { name: "task-disabled", disabled: true, internal: { - basePath: "/foo", + basePath: garden.projectRoot, }, timeout: DEFAULT_RUN_TIMEOUT_SEC, spec: { diff --git a/core/test/unit/src/config-graph.ts b/core/test/unit/src/config-graph.ts index c612332961..e082aaa076 100644 --- a/core/test/unit/src/config-graph.ts +++ b/core/test/unit/src/config-graph.ts @@ -154,15 +154,6 @@ describe("ConfigGraph (action-based configs)", () => { const buildActions = configGraph.getBuilds() expect(getNames(buildActions).sort()).to.eql(["build-1", "build-2", "build-3"]) - - const spec1 = buildActions[0].getConfig("spec") - expect(spec1.buildCommand).to.eql(["echo", "build-1", "ok"]) - - const spec2 = buildActions[1].getConfig("spec") - expect(spec2.buildCommand).to.eql(["echo", "build-2", "ok"]) - - const spec3 = buildActions[2].getConfig("spec") - expect(spec3.buildCommand).to.eql(["echo", "build-3", "ok"]) }) it("should optionally return specified Build actions in the context", async () => { @@ -222,15 +213,6 @@ describe("ConfigGraph (action-based configs)", () => { const deployActions = configGraph.getDeploys() expect(getNames(deployActions).sort()).to.eql(["deploy-1", "deploy-2", "deploy-3"]) - - const spec1 = deployActions[0].getConfig("spec") - expect(spec1.deployCommand).to.eql(["echo", "deploy-1", "ok"]) - - const spec2 = deployActions[1].getConfig("spec") - expect(spec2.deployCommand).to.eql(["echo", "deploy-2", "ok"]) - - const spec3 = deployActions[2].getConfig("spec") - expect(spec3.deployCommand).to.eql(["echo", "deploy-3", "ok"]) }) it("should optionally return specified Deploy actions in the context", async () => { @@ -290,15 +272,6 @@ describe("ConfigGraph (action-based configs)", () => { const runActions = configGraph.getRuns() expect(getNames(runActions).sort()).to.eql(["run-1", "run-2", "run-3"]) - - const spec1 = runActions[0].getConfig("spec") - expect(spec1.runCommand).to.eql(["echo", "run-1", "ok"]) - - const spec2 = runActions[1].getConfig("spec") - expect(spec2.runCommand).to.eql(["echo", "run-2", "ok"]) - - const spec3 = runActions[2].getConfig("spec") - expect(spec3.runCommand).to.eql(["echo", "run-3", "ok"]) }) it("should optionally return specified Run actions in the context", async () => { @@ -358,15 +331,6 @@ describe("ConfigGraph (action-based configs)", () => { const testActions = configGraph.getTests() expect(getNames(testActions).sort()).to.eql(["test-1", "test-2", "test-3"]) - - const spec1 = testActions[0].getConfig("spec") - expect(spec1.testCommand).to.eql(["echo", "test-1", "ok"]) - - const spec2 = testActions[1].getConfig("spec") - expect(spec2.testCommand).to.eql(["echo", "test-2", "ok"]) - - const spec3 = testActions[2].getConfig("spec") - expect(spec3.testCommand).to.eql(["echo", "test-3", "ok"]) }) it("should optionally return specified Test actions in the context", async () => { diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index c75da2e2cd..a3a6ec4ffe 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -4484,7 +4484,11 @@ describe("Garden", () => { } garden.treeCache.set(garden.log, ["moduleVersions", config.name], version, getModuleCacheContext(config)) - const result = await garden.resolveModuleVersion(garden.log, config, []) + const result = await garden.resolveModuleVersion({ + log: garden.log, + moduleConfig: config, + moduleDependencies: [], + }) expect(result).to.eql(version) }) @@ -4501,7 +4505,11 @@ describe("Garden", () => { files: [], }) - const result = await garden.resolveModuleVersion(garden.log, config, []) + const result = await garden.resolveModuleVersion({ + log: garden.log, + moduleConfig: config, + moduleDependencies: [], + }) expect(result.versionString).not.to.eql( config.version.versionString, @@ -4520,7 +4528,12 @@ describe("Garden", () => { } garden.treeCache.set(garden.log, ["moduleVersions", config.name], version, getModuleCacheContext(config)) - const result = await garden.resolveModuleVersion(garden.log, config, [], true) + const result = await garden.resolveModuleVersion({ + log: garden.log, + moduleConfig: config, + moduleDependencies: [], + force: true, + }) expect(result).to.not.eql(version) }) @@ -4549,7 +4562,11 @@ describe("Garden", () => { it("should return module version if there are no dependencies", async () => { const module = await gardenA.resolveModule("module-a") gardenA.vcs = handlerA - const result = await gardenA.resolveModuleVersion(gardenA.log, module, []) + const result = await gardenA.resolveModuleVersion({ + log: gardenA.log, + moduleConfig: module, + moduleDependencies: [], + }) const treeVersion = await handlerA.getTreeVersion({ log: gardenA.log, @@ -4596,7 +4613,11 @@ describe("Garden", () => { const treeVersionC: TreeVersion = { contentHash: versionStringC, files: [] } handlerA.setTestTreeVersion(moduleC.path, treeVersionC) - const gardenResolvedModuleVersion = await gardenA.resolveModuleVersion(gardenA.log, moduleC, [moduleA, moduleB]) + const gardenResolvedModuleVersion = await gardenA.resolveModuleVersion({ + log: gardenA.log, + moduleConfig: moduleC, + moduleDependencies: [moduleA, moduleB], + }) expect(gardenResolvedModuleVersion.versionString).to.equal( getModuleVersionString(moduleC, { ...treeVersionC, name: "module-c" }, [ @@ -4612,15 +4633,15 @@ describe("Garden", () => { it("should not include module's garden.yml in version file list", async () => { const moduleConfig = await gardenA.resolveModule("module-a") - const version = await gardenA.resolveModuleVersion(gardenA.log, moduleConfig, []) + const version = await gardenA.resolveModuleVersion({ log: gardenA.log, moduleConfig, moduleDependencies: [] }) expect(version.files).to.not.include(moduleConfig.configPath!) }) it("should be affected by changes to the module's config", async () => { const moduleConfig = await gardenA.resolveModule("module-a") - const version1 = await gardenA.resolveModuleVersion(gardenA.log, moduleConfig, []) + const version1 = await gardenA.resolveModuleVersion({ log: gardenA.log, moduleConfig, moduleDependencies: [] }) moduleConfig.name = "foo" - const version2 = await gardenA.resolveModuleVersion(gardenA.log, moduleConfig, []) + const version2 = await gardenA.resolveModuleVersion({ log: gardenA.log, moduleConfig, moduleDependencies: [] }) expect(version1).to.not.eql(version2) }) @@ -4632,9 +4653,17 @@ describe("Garden", () => { const orgConfig = await readFile(configPath) try { - const version1 = await gardenA.resolveModuleVersion(garden.log, moduleConfigA1, []) + const version1 = await gardenA.resolveModuleVersion({ + log: garden.log, + moduleConfig: moduleConfigA1, + moduleDependencies: [], + }) await writeFile(configPath, orgConfig + "\n---") - const version2 = await gardenA.resolveModuleVersion(garden.log, moduleConfigA1, []) + const version2 = await gardenA.resolveModuleVersion({ + log: garden.log, + moduleConfig: moduleConfigA1, + moduleDependencies: [], + }) expect(version1).to.eql(version2) } finally { await writeFile(configPath, orgConfig) diff --git a/core/test/unit/src/outputs.ts b/core/test/unit/src/outputs.ts index 4d11e1c93e..fd4adf00e1 100644 --- a/core/test/unit/src/outputs.ts +++ b/core/test/unit/src/outputs.ts @@ -163,7 +163,7 @@ describe("resolveProjectOutputs", () => { name: "test", type: "test", internal: { - basePath: "asd", + basePath: garden.projectRoot, }, kind: "Deploy", spec: { @@ -231,7 +231,7 @@ describe("resolveProjectOutputs", () => { name: "test", type: "test", internal: { - basePath: "asd", + basePath: garden.projectRoot, }, kind: "Run", spec: { diff --git a/core/test/unit/src/plugins/container/container.ts b/core/test/unit/src/plugins/container/container.ts index a972e3a3fa..5ad869aa9c 100644 --- a/core/test/unit/src/plugins/container/container.ts +++ b/core/test/unit/src/plugins/container/container.ts @@ -238,7 +238,14 @@ describe("plugins.container", () => { describe("version calculations", () => { async function getTestModule(moduleConfig: ContainerModuleConfig) { - return moduleFromConfig({ garden, log, config: moduleConfig, buildDependencies: [], forceVersion: true }) + return moduleFromConfig({ + garden, + log, + config: moduleConfig, + buildDependencies: [], + forceVersion: true, + scanRoot: garden.projectRoot, + }) } it("has same build version if nothing is changed", async () => { diff --git a/core/test/unit/src/router/test.ts b/core/test/unit/src/router/test.ts index c033ae5fde..05541f3c0a 100644 --- a/core/test/unit/src/router/test.ts +++ b/core/test/unit/src/router/test.ts @@ -29,15 +29,16 @@ describe("test actions", () => { const action = (await actionFromConfig({ garden, // rebuild config graph because the module config has been changed - graph: await garden.getConfigGraph({ emit: false, log: garden.log }), + graph, config: testConfig, log: garden.log, configsByKey: {}, router: await garden.getActionRouter(), mode: "default", linkedSources: {}, + scanRoot: garden.projectRoot, })) as TestAction - return await garden.resolveAction({ action, log: garden.log }) + return await garden.resolveAction({ action, log: garden.log, graph }) } before(async () => { diff --git a/core/test/unit/src/tasks/deploy.ts b/core/test/unit/src/tasks/deploy.ts index 56e4d6ec93..cedb74d9fe 100644 --- a/core/test/unit/src/tasks/deploy.ts +++ b/core/test/unit/src/tasks/deploy.ts @@ -99,7 +99,7 @@ describe("DeployTask", () => { type: "test", kind: "Deploy", internal: { - basePath: "foo", + basePath: garden.projectRoot, }, dependencies: [ { kind: "Deploy", name: "dep-deploy" }, @@ -116,7 +116,7 @@ describe("DeployTask", () => { type: "test", kind: "Deploy", internal: { - basePath: "foo", + basePath: garden.projectRoot, }, dependencies: [], disabled: false, @@ -132,7 +132,7 @@ describe("DeployTask", () => { disabled: false, timeout: 10, internal: { - basePath: "./", + basePath: garden.projectRoot, }, spec: { log: "test output", diff --git a/core/test/unit/src/vcs/git-repo.ts b/core/test/unit/src/vcs/git-repo.ts new file mode 100644 index 0000000000..8cd4d043a8 --- /dev/null +++ b/core/test/unit/src/vcs/git-repo.ts @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { GitRepoHandler } from "../../../../src/vcs/git-repo" +import { commonGitHandlerTests } from "./git" + +describe("GitRepoHandler", () => { + describe("getFiles", () => commonGitHandlerTests(GitRepoHandler)) +}) diff --git a/core/test/unit/src/vcs/git.ts b/core/test/unit/src/vcs/git.ts index 2743512d10..83c12b2444 100644 --- a/core/test/unit/src/vcs/git.ts +++ b/core/test/unit/src/vcs/git.ts @@ -13,14 +13,16 @@ import { createFile, ensureSymlink, lstat, mkdir, mkdirp, realpath, remove, syml import { basename, join, relative, resolve } from "path" import { expectError, makeTestGardenA, TestGarden } from "../../../helpers" -import { getCommitIdFromRefList, GitHandler, parseGitUrl } from "../../../../src/vcs/git" +import { getCommitIdFromRefList, GitCli, GitHandler, parseGitUrl } from "../../../../src/vcs/git" import { Log } from "../../../../src/logger/log-entry" import { hashRepoUrl } from "../../../../src/util/ext-source-util" import { deline } from "../../../../src/util/string" import { uuidv4 } from "../../../../src/util/random" +import { VcsHandlerParams } from "../../../../src/vcs/vcs" +import { repoRoot } from "../../../../src/util/testing" // Overriding this to make sure any ignorefile name is respected -const defaultIgnoreFilename = ".testignore" +export const defaultIgnoreFilename = ".testignore" async function getCommitMsg(repoPath: string) { const res = (await execa("git", ["log", "-1", "--pretty=%B"], { cwd: repoPath })).stdout @@ -42,7 +44,7 @@ async function createGitTag(tag: string, message: string, repoPath: string) { await execa("git", ["tag", "-a", tag, "-m", message], { cwd: repoPath }) } -async function makeTempGitRepo() { +export async function makeTempGitRepo() { const tmpDir = await tmp.dir({ unsafeCleanup: true }) const tmpPath = await realpath(tmpDir.path) await execa("git", ["init", "--initial-branch=main"], { cwd: tmpPath }) @@ -57,11 +59,15 @@ async function addToIgnore(tmpPath: string, pathToExclude: string, ignoreFilenam await writeFile(gardenignorePath, pathToExclude) } -describe("GitHandler", () => { +async function getGitHash(git: GitCli, path: string) { + return (await git("hash-object", path))[0] +} + +export const commonGitHandlerTests = (handlerCls: new (params: VcsHandlerParams) => GitHandler) => { let garden: TestGarden let tmpDir: tmp.DirectoryResult let tmpPath: string - let git: any + let git: GitCli let handler: GitHandler let log: Log @@ -70,94 +76,23 @@ describe("GitHandler", () => { log = garden.log tmpDir = await makeTempGitRepo() tmpPath = await realpath(tmpDir.path) - handler = new GitHandler({ + handler = new handlerCls({ garden, projectRoot: tmpPath, gardenDirPath: join(tmpPath, ".garden"), ignoreFile: defaultIgnoreFilename, cache: garden.treeCache, }) - git = (handler).gitCli(log, tmpPath) + git = handler.gitCli(log, tmpPath) }) afterEach(async () => { await tmpDir.cleanup() }) - async function getGitHash(path: string) { - return (await git("hash-object", path))[0] - } - - describe("toGitConfigCompatiblePath", () => { - it("should return an unmodified path in Linux", async () => { - const path = "/home/user/repo" - expect(handler.toGitConfigCompatiblePath(path, "linux")).to.equal(path) - }) - - it("should return an unmodified path in macOS", async () => { - const path = "/Users/user/repo" - expect(handler.toGitConfigCompatiblePath(path, "darwin")).to.equal(path) - }) - - it("should return a modified and corrected path in Windows", async () => { - const path = "C:\\Users\\user\\repo" - const expectedPath = "C:/Users/user/repo" - expect(handler.toGitConfigCompatiblePath(path, "win32")).to.equal(expectedPath) - }) - }) - - describe("getRepoRoot", () => { - it("should return the repo root if it is the same as the given path", async () => { - const path = tmpPath - expect(await handler.getRepoRoot(log, path)).to.equal(tmpPath) - }) - - it("should return the nearest repo root, given a subpath of that repo", async () => { - const dirPath = join(tmpPath, "dir") - await mkdir(dirPath) - expect(await handler.getRepoRoot(log, dirPath)).to.equal(tmpPath) - }) - - it("should throw a nice error when given a path outside of a repo", async () => { - await expectError( - () => handler.getRepoRoot(log, "/tmp"), - (err) => - expect(err.message).to.equal(deline` - Path /tmp is not in a git repository root. Garden must be run from within a git repo. - Please run \`git init\` if you're starting a new project and repository, or move the project to - an existing repository, and try again. - `) - ) - }) - }) - - describe("getPathInfo", () => { - it("should return empty strings with no commits in repo", async () => { - const path = tmpPath - const { branch, commitHash } = await handler.getPathInfo(log, path) - expect(branch).to.equal("") - expect(commitHash).to.equal("") - }) - - it("should return the current branch name when there are commits in the repo", async () => { - const path = tmpPath - await commit("init", tmpPath) - const { branch } = await handler.getPathInfo(log, path) - expect(branch).to.equal("main") - }) - - it("should return empty strings when given a path outside of a repo", async () => { - const path = tmpPath - const { branch, commitHash, originUrl } = await handler.getPathInfo(log, path) - expect(branch).to.equal("") - expect(commitHash).to.equal("") - expect(originUrl).to.equal("") - }) - }) - describe("getFiles", () => { it("should work with no commits in repo", async () => { - expect(await handler.getFiles({ path: tmpPath, log })).to.eql([]) + expect(await handler.getFiles({ path: tmpPath, scanRoot: undefined, log })).to.eql([]) }) it("should return tracked files as absolute paths with hash", async () => { @@ -168,9 +103,9 @@ describe("GitHandler", () => { await git("add", ".") await git("commit", "-m", "foo") - const hash = await getGitHash(path) + const hash = await getGitHash(git, path) - expect(await handler.getFiles({ path: tmpPath, log })).to.eql([{ path, hash }]) + expect(await handler.getFiles({ path: tmpPath, scanRoot: undefined, log })).to.eql([{ path, hash }]) }) it("should return the correct hash on a modified file", async () => { @@ -182,9 +117,9 @@ describe("GitHandler", () => { await writeFile(path, "my change") - const hash = await getGitHash(path) + const hash = await getGitHash(git, path) - expect(await handler.getFiles({ path: tmpPath, log })).to.eql([{ path, hash }]) + expect(await handler.getFiles({ path: tmpPath, scanRoot: undefined, log })).to.eql([{ path, hash }]) }) const dirContexts = [ @@ -203,11 +138,11 @@ describe("GitHandler", () => { await git("add", ".") await git("commit", "-m", "foo") - await writeFile(filePath, "my change") - const beforeHash = (await handler.getFiles({ path: dirPath, log }))[0].hash + await handler.writeFile(log, filePath, "my change") + const beforeHash = (await handler.getFiles({ path: dirPath, scanRoot: undefined, log }))[0].hash - await writeFile(filePath, "ch-ch-ch-ch-changes") - const afterHash = (await handler.getFiles({ path: dirPath, log }))[0].hash + await handler.writeFile(log, filePath, "ch-ch-ch-ch-changes") + const afterHash = (await handler.getFiles({ path: dirPath, scanRoot: undefined, log }))[0].hash expect(beforeHash).to.not.eql(afterHash) }) @@ -217,9 +152,9 @@ describe("GitHandler", () => { const path = join(dirPath, "foo.txt") await createFile(path) - const hash = await getGitHash(path) + const hash = await getGitHash(git, path) - expect(await handler.getFiles({ path: dirPath, log })).to.eql([{ path, hash }]) + expect(await handler.getFiles({ path: dirPath, scanRoot: undefined, log })).to.eql([{ path, hash }]) }) }) } @@ -230,9 +165,9 @@ describe("GitHandler", () => { await mkdir(dirPath) await createFile(path) - const hash = await getGitHash(path) + const hash = await getGitHash(git, path) - expect(await handler.getFiles({ path: dirPath, log })).to.eql([{ path, hash }]) + expect(await handler.getFiles({ path: dirPath, scanRoot: undefined, log })).to.eql([{ path, hash }]) }) it("should work with tracked files with spaces in the name", async () => { @@ -241,9 +176,9 @@ describe("GitHandler", () => { await git("add", path) await git("commit", "-m", "foo") - const hash = await getGitHash(path) + const hash = await getGitHash(git, path) - expect(await handler.getFiles({ path: tmpPath, log })).to.eql([{ path, hash }]) + expect(await handler.getFiles({ path: tmpPath, scanRoot: undefined, log })).to.eql([{ path, hash }]) }) it("should work with tracked+modified files with spaces in the name", async () => { @@ -254,9 +189,9 @@ describe("GitHandler", () => { await writeFile(path, "fooooo") - const hash = await getGitHash(path) + const hash = await getGitHash(git, path) - expect(await handler.getFiles({ path: tmpPath, log })).to.eql([{ path, hash }]) + expect(await handler.getFiles({ path: tmpPath, scanRoot: undefined, log })).to.eql([{ path, hash }]) }) it("should gracefully skip files that are deleted after having been committed", async () => { @@ -267,38 +202,40 @@ describe("GitHandler", () => { await remove(filePath) - expect(await handler.getFiles({ path: tmpPath, log })).to.eql([]) + expect(await handler.getFiles({ path: tmpPath, scanRoot: undefined, log })).to.eql([]) }) it("should work with untracked files with spaces in the name", async () => { const path = join(tmpPath, "my file.txt") await createFile(path) - const hash = await getGitHash(path) + const hash = await getGitHash(git, path) - expect(await handler.getFiles({ path: tmpPath, log })).to.eql([{ path, hash }]) + expect(await handler.getFiles({ path: tmpPath, scanRoot: undefined, log })).to.eql([{ path, hash }]) }) it("should return nothing if include: []", async () => { const path = resolve(tmpPath, "foo.txt") await createFile(path) - expect(await handler.getFiles({ path: tmpPath, include: [], log })).to.eql([]) + expect(await handler.getFiles({ path: tmpPath, scanRoot: undefined, include: [], log })).to.eql([]) }) it("should filter out files that don't match the include filter, if specified", async () => { const path = resolve(tmpPath, "foo.txt") await createFile(path) - expect(await handler.getFiles({ path: tmpPath, include: ["bar.*"], log })).to.eql([]) + expect(await handler.getFiles({ path: tmpPath, scanRoot: undefined, include: ["bar.*"], log })).to.eql([]) }) it("should include files that match the include filter, if specified", async () => { const path = resolve(tmpPath, "foo.txt") await createFile(path) - const hash = await getGitHash(path) + const hash = await getGitHash(git, path) - expect(await handler.getFiles({ path: tmpPath, include: ["foo.*"], exclude: [], log })).to.eql([{ path, hash }]) + expect( + await handler.getFiles({ path: tmpPath, scanRoot: undefined, include: ["foo.*"], exclude: [], log }) + ).to.eql([{ path, hash }]) }) it("should include a directory that's explicitly included by exact name", async () => { @@ -307,26 +244,30 @@ describe("GitHandler", () => { await mkdir(subdir) const path = resolve(tmpPath, subdirName, "foo.txt") await createFile(path) - const hash = await getGitHash(path) + const hash = await getGitHash(git, path) - expect(await handler.getFiles({ path: tmpPath, include: [subdirName], exclude: [], log })).to.eql([ - { path, hash }, - ]) + expect( + await handler.getFiles({ path: tmpPath, scanRoot: undefined, include: [subdirName], exclude: [], log }) + ).to.eql([{ path, hash }]) }) it("should include hidden files that match the include filter, if specified", async () => { const path = resolve(tmpPath, ".foo") await createFile(path) - const hash = await getGitHash(path) + const hash = await getGitHash(git, path) - expect(await handler.getFiles({ path: tmpPath, include: ["*"], exclude: [], log })).to.eql([{ path, hash }]) + expect(await handler.getFiles({ path: tmpPath, scanRoot: undefined, include: ["*"], exclude: [], log })).to.eql([ + { path, hash }, + ]) }) it("should filter out files that match the exclude filter, if specified", async () => { const path = resolve(tmpPath, "foo.txt") await createFile(path) - expect(await handler.getFiles({ path: tmpPath, include: [], exclude: ["foo.*"], log })).to.eql([]) + expect( + await handler.getFiles({ path: tmpPath, scanRoot: undefined, include: [], exclude: ["foo.*"], log }) + ).to.eql([]) }) it("should respect include and exclude patterns, if both are specified", async () => { @@ -345,6 +286,7 @@ describe("GitHandler", () => { include: ["module-a/**/*"], exclude: ["**/*.txt"], log, + scanRoot: undefined, }) ).map((f) => f.path) @@ -357,7 +299,7 @@ describe("GitHandler", () => { await createFile(path) await addToIgnore(tmpPath, name) - const files = (await handler.getFiles({ path: tmpPath, exclude: [], log })).filter( + const files = (await handler.getFiles({ path: tmpPath, scanRoot: undefined, exclude: [], log })).filter( (f) => !f.path.includes(defaultIgnoreFilename) ) @@ -373,7 +315,7 @@ describe("GitHandler", () => { await git("add", path) await git("commit", "-m", "foo") - const files = (await handler.getFiles({ path: tmpPath, exclude: [], log })).filter( + const files = (await handler.getFiles({ path: tmpPath, scanRoot: undefined, exclude: [], log })).filter( (f) => !f.path.includes(defaultIgnoreFilename) ) @@ -388,7 +330,7 @@ describe("GitHandler", () => { await git("add", ".") await git("commit", "-m", "foo") - const hash = await getGitHash(path) + const hash = await getGitHash(git, path) const _handler = new GitHandler({ garden, @@ -398,7 +340,7 @@ describe("GitHandler", () => { cache: garden.treeCache, }) - expect(await _handler.getFiles({ path: tmpPath, log })).to.eql([{ path, hash }]) + expect(await _handler.getFiles({ path: tmpPath, scanRoot: undefined, log })).to.eql([{ path, hash }]) }) it("should include a relative symlink within the path", async () => { @@ -409,12 +351,18 @@ describe("GitHandler", () => { await createFile(filePath) await symlink(fileName, symlinkPath) - const files = (await handler.getFiles({ path: tmpPath, exclude: [], log })).map((f) => f.path) + const files = (await handler.getFiles({ path: tmpPath, scanRoot: undefined, exclude: [], log })).map( + (f) => f.path + ) expect(files).to.eql([filePath, symlinkPath]) }) - it("should exclude a relative symlink that points outside the path", async () => { + it("should exclude a relative symlink that points outside repo root", async () => { const subPath = resolve(tmpPath, "subdir") + await mkdirp(subPath) + + const _git = handler.gitCli(log, subPath) + await _git("init") const fileName = "foo" const filePath = resolve(tmpPath, fileName) @@ -423,7 +371,9 @@ describe("GitHandler", () => { await createFile(filePath) await ensureSymlink(join("..", fileName), symlinkPath) - const files = (await handler.getFiles({ path: subPath, exclude: [], log })).map((f) => f.path) + const files = (await handler.getFiles({ path: subPath, scanRoot: undefined, exclude: [], log })).map( + (f) => f.path + ) expect(files).to.eql([]) }) @@ -435,14 +385,16 @@ describe("GitHandler", () => { await createFile(filePath) await symlink(filePath, symlinkPath) - const files = (await handler.getFiles({ path: tmpPath, exclude: [], log })).map((f) => f.path) + const files = (await handler.getFiles({ path: tmpPath, scanRoot: undefined, exclude: [], log })).map( + (f) => f.path + ) expect(files).to.eql([filePath]) }) it("gracefully aborts if given path doesn't exist", async () => { const path = resolve(tmpPath, "foo") - const files = (await handler.getFiles({ path, exclude: [], log })).map((f) => f.path) + const files = (await handler.getFiles({ path, scanRoot: undefined, exclude: [], log })).map((f) => f.path) expect(files).to.eql([]) }) @@ -450,10 +402,18 @@ describe("GitHandler", () => { const path = resolve(tmpPath, "foo") await createFile(path) - const files = (await handler.getFiles({ path, exclude: [], log })).map((f) => f.path) + const files = (await handler.getFiles({ path, scanRoot: undefined, exclude: [], log })).map((f) => f.path) expect(files).to.eql([]) }) + context("large repo", () => { + it("does its thing in a reasonable amount of time", async () => { + const scanRoot = join(repoRoot, "examples") + const path = join(scanRoot, "demo-project") + await handler.getFiles({ path, scanRoot, exclude: [], log }) + }) + }) + context("path contains a submodule", () => { let submodule: tmp.DirectoryResult let submodulePath: string @@ -477,15 +437,15 @@ describe("GitHandler", () => { }) it("should include tracked files in submodules", async () => { - const files = await handler.getFiles({ path: tmpPath, log }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log }) const paths = files.map((f) => relative(tmpPath, f.path)) expect(paths).to.eql([".gitmodules", join("sub", initFile)]) }) - it("should work if submodule is not initialized and not include any files", async () => { + it("should work if submodule is not initialized and doesn't include any files", async () => { await execa("git", ["submodule", "deinit", "--all"], { cwd: tmpPath }) - const files = await handler.getFiles({ path: tmpPath, log }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log }) const paths = files.map((f) => relative(tmpPath, f.path)) expect(paths).to.eql([".gitmodules", "sub"]) @@ -494,7 +454,7 @@ describe("GitHandler", () => { it("should work if submodule is initialized but not updated", async () => { await execa("git", ["submodule", "deinit", "--all"], { cwd: tmpPath }) await execa("git", ["submodule", "init"], { cwd: tmpPath }) - const files = await handler.getFiles({ path: tmpPath, log }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log }) const paths = files.map((f) => relative(tmpPath, f.path)) expect(paths).to.eql([".gitmodules", "sub"]) @@ -504,7 +464,7 @@ describe("GitHandler", () => { const path = join(tmpPath, "sub", "x.txt") await createFile(path) - const files = await handler.getFiles({ path: tmpPath, log }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log }) const paths = files.map((f) => relative(tmpPath, f.path)).sort() expect(paths).to.eql([".gitmodules", join("sub", initFile), join("sub", "x.txt")]) @@ -514,7 +474,7 @@ describe("GitHandler", () => { const path = join(tmpPath, "sub", "x.foo") await createFile(path) - const files = await handler.getFiles({ path: tmpPath, log, include: ["**/*.txt"] }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log, include: ["**/*.txt"] }) const paths = files.map((f) => relative(tmpPath, f.path)).sort() expect(paths).to.not.include(join("sub", path)) @@ -525,7 +485,7 @@ describe("GitHandler", () => { const path = join(tmpPath, "sub", "x.foo") await createFile(path) - const files = await handler.getFiles({ path: tmpPath, log, exclude: ["sub/*.txt"] }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log, exclude: ["sub/*.txt"] }) const paths = files.map((f) => relative(tmpPath, f.path)).sort() expect(paths).to.eql([".gitmodules", join("sub", "x.foo")]) @@ -535,7 +495,7 @@ describe("GitHandler", () => { const path = join(tmpPath, "sub", "x.foo") await createFile(path) - const files = await handler.getFiles({ path: tmpPath, log, include: ["./sub/*.txt"] }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log, include: ["./sub/*.txt"] }) const paths = files.map((f) => relative(tmpPath, f.path)).sort() expect(paths).to.not.include(join("sub", path)) @@ -543,7 +503,7 @@ describe("GitHandler", () => { }) it("should include the whole submodule contents when an include directly specifies its path", async () => { - const files = await handler.getFiles({ path: tmpPath, log, include: ["sub"] }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log, include: ["sub"] }) const paths = files.map((f) => relative(tmpPath, f.path)).sort() expect(paths).to.include(join("sub", initFile)) @@ -558,14 +518,14 @@ describe("GitHandler", () => { await createFile(path) await commit(relPath, submodulePath) - const files = await handler.getFiles({ path: tmpPath, log, include: ["sub/subdir"] }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log, include: ["sub/subdir"] }) const paths = files.map((f) => relative(tmpPath, f.path)).sort() expect(paths).to.eql([relPath]) }) it("should include the whole submodule when a surrounding include matches it", async () => { - const files = await handler.getFiles({ path: tmpPath, log, include: ["**/*"] }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log, include: ["**/*"] }) const paths = files.map((f) => relative(tmpPath, f.path)).sort() expect(paths).to.include(join("sub", initFile)) @@ -575,7 +535,7 @@ describe("GitHandler", () => { const subPath = join(tmpPath, "sub") await remove(subPath) - const files = await handler.getFiles({ path: tmpPath, log }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log }) const paths = files.map((f) => relative(tmpPath, f.path)) expect(paths).to.eql([".gitmodules"]) @@ -586,7 +546,7 @@ describe("GitHandler", () => { await remove(subPath) await createFile(subPath) - const files = await handler.getFiles({ path: tmpPath, log }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log }) const paths = files.map((f) => relative(tmpPath, f.path)) expect(paths).to.eql([".gitmodules"]) @@ -613,7 +573,7 @@ describe("GitHandler", () => { }) it("should include tracked files in nested submodules", async () => { - const files = await handler.getFiles({ path: tmpPath, log }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log }) const paths = files.map((f) => relative(tmpPath, f.path)).sort() expect(paths).to.eql([ @@ -629,7 +589,7 @@ describe("GitHandler", () => { const path = join(dir, "x.txt") await createFile(path) - const files = await handler.getFiles({ path: tmpPath, log }) + const files = await handler.getFiles({ path: tmpPath, scanRoot: undefined, log }) const paths = files.map((f) => relative(tmpPath, f.path)).sort() expect(paths).to.eql([ @@ -644,6 +604,73 @@ describe("GitHandler", () => { }) }) + describe("toGitConfigCompatiblePath", () => { + it("should return an unmodified path in Linux", async () => { + const path = "/home/user/repo" + expect(handler.toGitConfigCompatiblePath(path, "linux")).to.equal(path) + }) + + it("should return an unmodified path in macOS", async () => { + const path = "/Users/user/repo" + expect(handler.toGitConfigCompatiblePath(path, "darwin")).to.equal(path) + }) + + it("should return a modified and corrected path in Windows", async () => { + const path = "C:\\Users\\user\\repo" + const expectedPath = "C:/Users/user/repo" + expect(handler.toGitConfigCompatiblePath(path, "win32")).to.equal(expectedPath) + }) + }) + + describe("getRepoRoot", () => { + it("should return the repo root if it is the same as the given path", async () => { + const path = tmpPath + expect(await handler.getRepoRoot(log, path)).to.equal(tmpPath) + }) + + it("should return the nearest repo root, given a subpath of that repo", async () => { + const dirPath = join(tmpPath, "dir") + await mkdir(dirPath) + expect(await handler.getRepoRoot(log, dirPath)).to.equal(tmpPath) + }) + + it("should throw a nice error when given a path outside of a repo", async () => { + await expectError( + () => handler.getRepoRoot(log, "/tmp"), + (err) => + expect(err.message).to.equal(deline` + Path /tmp is not in a git repository root. Garden must be run from within a git repo. + Please run \`git init\` if you're starting a new project and repository, or move the project to + an existing repository, and try again. + `) + ) + }) + }) + + describe("getPathInfo", () => { + it("should return empty strings with no commits in repo", async () => { + const path = tmpPath + const { branch, commitHash } = await handler.getPathInfo(log, path) + expect(branch).to.equal("") + expect(commitHash).to.equal("") + }) + + it("should return the current branch name when there are commits in the repo", async () => { + const path = tmpPath + await commit("init", tmpPath) + const { branch } = await handler.getPathInfo(log, path) + expect(branch).to.equal("main") + }) + + it("should return empty strings when given a path outside of a repo", async () => { + const path = tmpPath + const { branch, commitHash, originUrl } = await handler.getPathInfo(log, path) + expect(branch).to.equal("") + expect(commitHash).to.equal("") + expect(originUrl).to.equal("") + }) + }) + describe("hashObject", () => { it("should return the same result as `git hash-object` for a file", async () => { const path = resolve(tmpPath, "foo.txt") @@ -651,7 +678,7 @@ describe("GitHandler", () => { await writeFile(path, "iogjeiojgeowigjewoijoeiw") const stats = await lstat(path) - const expected = await getGitHash(path) + const expected = await getGitHash(git, path) return new Promise((_resolve, reject) => { handler.hashObject(stats, path, (err, hash) => { @@ -959,6 +986,10 @@ describe("GitHandler", () => { }) }) }) +} + +describe("GitHandler", () => { + commonGitHandlerTests(GitHandler) }) describe("git", () => { diff --git a/core/test/unit/src/vcs/vcs.ts b/core/test/unit/src/vcs/vcs.ts index d58cdfe5c6..9ebdea6869 100644 --- a/core/test/unit/src/vcs/vcs.ts +++ b/core/test/unit/src/vcs/vcs.ts @@ -22,6 +22,7 @@ import { describeConfig, getConfigBasePath, getConfigFilePath, + GetTreeVersionParams, } from "../../../../src/vcs/vcs" import { makeTestGardenA, makeTestGarden, getDataDir, TestGarden, defaultModuleConfig } from "../../../helpers" import { expect } from "chai" @@ -34,7 +35,7 @@ import tmp from "tmp-promise" import { realpath, readFile, writeFile, rm, rename } from "fs-extra" import { DEFAULT_BUILD_TIMEOUT_SEC, GARDEN_VERSIONFILE_NAME, GardenApiVersion } from "../../../../src/constants" import { defaultDotIgnoreFile, fixedProjectExcludes } from "../../../../src/util/fs" -import { Log, createActionLog } from "../../../../src/logger/log-entry" +import { createActionLog } from "../../../../src/logger/log-entry" import { BaseActionConfig } from "../../../../src/actions/types" import { TreeCache } from "../../../../src/cache" @@ -58,8 +59,8 @@ export class TestVcsHandler extends VcsHandler { } } - async getTreeVersion({ log, projectName, config }: { log: Log; projectName: string; config: ModuleConfig }) { - return this.testTreeVersions[config.path] || super.getTreeVersion({ log, projectName, config }) + async getTreeVersion(params: GetTreeVersionParams) { + return this.testTreeVersions[getConfigBasePath(params.config)] || super.getTreeVersion(params) } setTestTreeVersion(path: string, version: TreeVersion) { diff --git a/docs/reference/project-config.md b/docs/reference/project-config.md index 2cdc7a0529..6a337f0320 100644 --- a/docs/reference/project-config.md +++ b/docs/reference/project-config.md @@ -119,7 +119,7 @@ proxy: # Note that the `GARDEN_PROXY_DEFAULT_ADDRESS` environment variable takes precedence over this value. hostname: localhost -# Control where to scan for configuration files in the project. +# Control where and how to scan for configuration files in the project. scan: # Specify a list of POSIX-style paths or globs that should be scanned for Garden configuration files. # @@ -154,6 +154,13 @@ scan: # details. exclude: + git: + # Choose how to perform scans of git repositories. The default (`subtree`) runs individual git scans on each + # action/module path. The `repo` mode scans entire repositories and then filters down to files matching the paths, + # includes and excludes for each action/module. This can be considerably more efficient for large projects with + # many actions/modules. + mode: subtree + # A list of output values that the project should export. These are exported by the `garden get outputs` command, as # well as when referencing a project as a sub-project within another project. # @@ -496,7 +503,7 @@ proxy: ### `scan` -Control where to scan for configuration files in the project. +Control where and how to scan for configuration files in the project. | Type | Required | | -------- | -------- | @@ -557,6 +564,24 @@ scan: - tmp/**/* ``` +### `scan.git` + +[scan](#scan) > git + +| Type | Required | +| -------- | -------- | +| `object` | No | + +### `scan.git.mode` + +[scan](#scan) > [git](#scangit) > mode + +Choose how to perform scans of git repositories. The default (`subtree`) runs individual git scans on each action/module path. The `repo` mode scans entire repositories and then filters down to files matching the paths, includes and excludes for each action/module. This can be considerably more efficient for large projects with many actions/modules. + +| Type | Allowed Values | Default | Required | +| -------- | ----------------- | ----------- | -------- | +| `string` | "repo", "subtree" | `"subtree"` | Yes | + ### `outputs[]` A list of output values that the project should export. These are exported by the `garden get outputs` command, as well as when referencing a project as a sub-project within another project.