diff --git a/docs/using-garden/configuration-files.md b/docs/using-garden/configuration-files.md index ef8f0e652c..07597ad6e3 100644 --- a/docs/using-garden/configuration-files.md +++ b/docs/using-garden/configuration-files.md @@ -399,6 +399,14 @@ name: my-project dotIgnoreFiles: [.gardenignore] ``` +#### Git submodules + +If you're using Git submodules in your project, please note the following: + +1. You may ignore submodules using .ignore files and include/exclude filters. If a submodule path _itself_ (that is, the path to the submodule directory, not its contents), matches one that is ignored by your .ignore files or exclude filters, or if you specify include filters and the submodule path does not match one of them, the module will not be scanned. +2. Include/exclude filters (both at the project and module level) are applied the same, whether a directory is a submodule or a normal directory. +3. _.ignore files are considered in the context of each git root_. This means that a .ignore file that's outside of a submodule will be completely ignored when scanning that submodule. This is by design, to be consistent with normal Git behavior. + ## Next steps We highly recommend reading the [Variables and Templating guide](./variables-and-templating.md) to understand how you can reference across different providers and modules, as well as to understand how to supply secret values to your configuration. diff --git a/garden-service/package-lock.json b/garden-service/package-lock.json index ab0642abb6..549e4dbf9a 100644 --- a/garden-service/package-lock.json +++ b/garden-service/package-lock.json @@ -960,6 +960,12 @@ "integrity": "sha512-Nd8y/5t/7CRakPYiyPzr/IAfYusy1FkcZYFEAcoMZkwpJv2n4Wm+olW+e7xBdHEXhOnWdG9ddbar0gqZWS4x5Q==", "dev": true }, + "@types/parse-git-config": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/parse-git-config/-/parse-git-config-3.0.0.tgz", + "integrity": "sha512-5C14/81ohSwjB5I0EweuG3qyn6CqgVOgk9orxHDwVvUdDbS4FMXANXnKz3CA3H2Jk2oa3vzMEpXtDQ5u0dCcTQ==", + "dev": true + }, "@types/path-is-inside": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/path-is-inside/-/path-is-inside-1.0.0.tgz", @@ -4805,6 +4811,11 @@ "assert-plus": "^1.0.0" } }, + "git-config-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/git-config-path/-/git-config-path-2.0.0.tgz", + "integrity": "sha512-qc8h1KIQbJpp+241id3GuAtkdyJ+IK+LIVtkiFTRKRrmddDzs3SI9CvP1QYmWBFvm1I/PWRwj//of8bgAc0ltA==" + }, "git-raw-commits": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/git-raw-commits/-/git-raw-commits-1.3.6.tgz", @@ -9215,6 +9226,15 @@ "path-root": "^0.1.1" } }, + "parse-git-config": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/parse-git-config/-/parse-git-config-3.0.0.tgz", + "integrity": "sha512-wXoQGL1D+2COYWCD35/xbiKma1Z15xvZL8cI25wvxzled58V51SJM04Urt/uznS900iQor7QO04SgdfT/XlbuA==", + "requires": { + "git-config-path": "^2.0.0", + "ini": "^1.3.5" + } + }, "parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", diff --git a/garden-service/package.json b/garden-service/package.json index c8a93a7218..f881efc082 100644 --- a/garden-service/package.json +++ b/garden-service/package.json @@ -89,6 +89,7 @@ "normalize-url": "^4.3.0", "p-queue": "^6.1.1", "p-retry": "^4.1.0", + "parse-git-config": "^3.0.0", "path-is-inside": "^1.0.2", "pluralize": "^8.0.0", "proper-url-join": "^2.0.1", @@ -152,6 +153,7 @@ "@types/node-emoji": "^1.8.1", "@types/node-forge": "^0.8.6", "@types/normalize-path": "^3.0.0", + "@types/parse-git-config": "^3.0.0", "@types/path-is-inside": "^1.0.0", "@types/pluralize": "0.0.29", "@types/prettyjson": "0.0.29", diff --git a/garden-service/src/garden.ts b/garden-service/src/garden.ts index 65c5505778..3a2cb85b8f 100644 --- a/garden-service/src/garden.ts +++ b/garden-service/src/garden.ts @@ -40,7 +40,7 @@ import { platform, arch } from "os" import { LogEntry } from "./logger/log-entry" import { EventBus } from "./events" import { Watcher } from "./watch" -import { findConfigPathsInPath, getConfigFilePath, getWorkingCopyId } from "./util/fs" +import { findConfigPathsInPath, getConfigFilePath, getWorkingCopyId, fixedExcludes } from "./util/fs" import { Provider, ProviderConfig, getProviderDependencies, defaultProvider } from "./config/provider" import { ResolveProviderTask } from "./tasks/resolve-provider" import { ActionHelper } from "./actions" @@ -226,7 +226,11 @@ export class Garden { // We always exclude the garden dir const gardenDirExcludePattern = `${relative(projectRoot, gardenDirPath)}/**/*` - const moduleExcludePatterns = [...((config.modules || {}).exclude || []), gardenDirExcludePattern] + const moduleExcludePatterns = [ + ...((config.modules || {}).exclude || []), + gardenDirExcludePattern, + ...fixedExcludes, + ] // Ensure the project root is in a git repo const vcs = new GitHandler(gardenDirPath, config.dotIgnoreFiles) diff --git a/garden-service/src/util/fs.ts b/garden-service/src/util/fs.ts index 03e4b9a17b..7c109b2362 100644 --- a/garden-service/src/util/fs.ts +++ b/garden-service/src/util/fs.ts @@ -22,7 +22,7 @@ import { LogEntry } from "../logger/log-entry" const VALID_CONFIG_FILENAMES = ["garden.yml", "garden.yaml"] const metadataFilename = "metadata.json" export const defaultDotIgnoreFiles = [".gitignore", ".gardenignore"] -export const fixedExcludes = [".git", ".garden/**/*", "debug-info*/**"] +export const fixedExcludes = [".git", ".gitmodules", ".garden/**/*", "debug-info*/**"] /* Warning: Don't make any async calls in the loop body when using this function, since this may cause @@ -116,7 +116,7 @@ export async function findConfigPathsInPath( { vcs: VcsHandler, dir: string, include?: string[], exclude?: string[], log: LogEntry }, ) { // TODO: we could make this lighter/faster using streaming - const files = await vcs.getFiles({ path: dir, include, exclude: [...exclude || [], ...fixedExcludes], log }) + const files = await vcs.getFiles({ path: dir, include, exclude: exclude || [], log }) return files .map(f => f.path) .filter(f => isConfigFilename(basename(f))) diff --git a/garden-service/src/vcs/git.ts b/garden-service/src/vcs/git.ts index 677f87adc4..ef069d22a5 100644 --- a/garden-service/src/vcs/git.ts +++ b/garden-service/src/vcs/git.ts @@ -7,7 +7,7 @@ */ import execa from "execa" -import { join, resolve } from "path" +import { join, resolve, relative } from "path" import { flatten } from "lodash" import { ensureDir, pathExists, stat, createReadStream } from "fs-extra" import { PassThrough } from "stream" @@ -21,6 +21,7 @@ import { matchPath } from "../util/fs" import { deline } from "../util/string" import { splitLast } from "../util/util" import { LogEntry } from "../logger/log-entry" +import parseGitConfig from "parse-git-config" export function getCommitIdFromRefList(refList: string[]): string { try { @@ -48,6 +49,11 @@ interface GitCli { (...args: string[]): Promise } +interface Submodule { + path: string + url: string +} + // TODO Consider moving git commands to separate (and testable) functions export class GitHandler extends VcsHandler { name = "git" @@ -113,6 +119,9 @@ export class GitHandler extends VcsHandler { )), )) + // List all submodule paths in the current repo + const submodulePaths = (await this.getSubmodules(gitRoot)).map(s => join(gitRoot, s.path)) + // We run ls-files for each ignoreFile and do a manual set-intersection (by counting elements in an object) // in order to optimize the flow. const paths: { [path: string]: number } = {} @@ -156,7 +165,10 @@ export class GitHandler extends VcsHandler { // We push to the output array when all ls-files commands "agree" that it should be included, // and it passes through the include/exclude filters. - if (paths[resolvedPath] === this.ignoreFiles.length && matchPath(filePath, include, exclude)) { + if ( + paths[resolvedPath] === this.ignoreFiles.length + && (matchPath(filePath, include, exclude) || submodulePaths.includes(resolvedPath)) + ) { files.push({ path: resolvedPath, hash }) } } @@ -181,8 +193,20 @@ export class GitHandler extends VcsHandler { } }) + // Resolve submodules + const withSubmodules = flatten(await Bluebird.map(files, async (f) => { + if (submodulePaths.includes(f.path)) { + // This path is a submodule, so we recursively call getFiles for that path again. + // Note: We apply include/exclude filters after listing files from submodule + return (await this.getFiles({ log, path: f.path, exclude: [] })) + .filter(submoduleFile => matchPath(relative(path, submoduleFile.path), include, exclude)) + } else { + return [f] + } + })) + // Make sure we have a fresh hash for each file - return Bluebird.map(files, async (f) => { + return Bluebird.map(withSubmodules, async (f) => { const resolvedPath = resolve(path, f.path) if (!f.hash || modified.has(resolvedPath)) { // If we can't compute the hash, i.e. the file is gone, we filter it out below @@ -289,4 +313,22 @@ export class GitHandler extends VcsHandler { createReadStream(path).pipe(stream) return output } + + private async getSubmodules(gitRoot: string) { + const submodules: Submodule[] = [] + const gitmodulesPath = join(gitRoot, ".gitmodules") + + if (await pathExists(gitmodulesPath)) { + const parsed = await parseGitConfig({ cwd: gitRoot, path: ".gitmodules" }) + + for (const [key, spec] of Object.entries(parsed || {}) as any) { + if (!key.startsWith("submodule")) { + continue + } + spec.path && submodules.push(spec) + } + } + + return submodules + } } diff --git a/garden-service/test/unit/src/vcs/git.ts b/garden-service/test/unit/src/vcs/git.ts index 1f724ab783..62cd084f27 100644 --- a/garden-service/test/unit/src/vcs/git.ts +++ b/garden-service/test/unit/src/vcs/git.ts @@ -11,11 +11,10 @@ import { expect } from "chai" import tmp from "tmp-promise" import uuid from "uuid" import { createFile, writeFile, realpath, mkdir, remove, symlink } from "fs-extra" -import { join, resolve, basename } from "path" +import { join, resolve, basename, relative } from "path" import { expectError, makeTestGardenA } from "../../../helpers" import { getCommitIdFromRefList, parseGitUrl, GitHandler } from "../../../../src/vcs/git" -import { fixedExcludes } from "../../../../src/util/fs" import { LogEntry } from "../../../../src/logger/log-entry" import { hashRepoUrl } from "../../../../src/util/ext-source-util" import { deline } from "../../../../src/util/string" @@ -30,20 +29,19 @@ async function getCommitMsg(repoPath: string) { async function commit(msg: string, repoPath: string) { // Ensure master contains changes when commiting - const uniqueFilename = uuid.v4() - const filePath = join(repoPath, `${uniqueFilename}.txt`) + const uniqueFilename = `${uuid.v4()}.txt` + const filePath = join(repoPath, uniqueFilename) await createFile(filePath) await execa("git", ["add", filePath], { cwd: repoPath }) await execa("git", ["commit", "-m", msg], { cwd: repoPath }) + return uniqueFilename } -async function makeTempGitRepo(initCommitMsg: string = "test commit") { +async function makeTempGitRepo() { const tmpDir = await tmp.dir({ unsafeCleanup: true }) const tmpPath = await realpath(tmpDir.path) await execa("git", ["init"], { cwd: tmpPath }) - await commit(initCommitMsg, tmpPath) - return tmpDir } @@ -64,11 +62,10 @@ describe("GitHandler", () => { beforeEach(async () => { const garden = await makeTestGardenA() log = garden.log - tmpDir = await tmp.dir({ unsafeCleanup: true }) + tmpDir = await makeTempGitRepo() tmpPath = await realpath(tmpDir.path) handler = new GitHandler(tmpPath, [defaultIgnoreFilename]) git = (handler).gitCli(log, tmpPath) - await git("init") }) afterEach(async () => { @@ -320,20 +317,6 @@ describe("GitHandler", () => { expect(files).to.eql([]) }) - it("should exclude files that are exclude by default", async () => { - for (const exclude of fixedExcludes) { - const name = "foo.txt" - const updatedExclude = exclude.replace("**", "a-folder").replace("*", "-a-value/sisis") - const path = resolve(join(tmpPath, updatedExclude), name) - await createFile(path) - } - - const files = (await handler.getFiles({ path: tmpPath, exclude: [...fixedExcludes], log })) - .filter(f => !f.path.includes(defaultIgnoreFilename)) - - expect(files).to.eql([]) - }) - it("should exclude an untracked symlink to a directory", async () => { const tmpDir2 = await tmp.dir({ unsafeCleanup: true }) const tmpPathB = await realpath(tmpDir2.path) @@ -348,6 +331,110 @@ describe("GitHandler", () => { expect(files).to.eql([]) }) + + context("path contains a submodule", () => { + let submodule: tmp.DirectoryResult + let submodulePath: string + let initFile: string + + beforeEach(async () => { + submodule = await makeTempGitRepo() + submodulePath = await realpath(submodule.path) + initFile = await commit("init", submodulePath) + + await execa("git", ["submodule", "add", submodulePath, "sub"], { cwd: tmpPath }) + await execa("git", ["commit", "-m", "add submodule"], { cwd: tmpPath }) + }) + + afterEach(async () => { + await submodule.cleanup() + }) + + it("should include tracked files in submodules", async () => { + const files = await handler.getFiles({ path: tmpPath, log }) + const paths = files.map(f => relative(tmpPath, f.path)) + + expect(paths).to.eql([".gitmodules", join("sub", initFile)]) + }) + + it("should include untracked files in submodules", async () => { + const path = join(tmpPath, "sub", "x.txt") + await createFile(path) + + const files = await handler.getFiles({ path: tmpPath, log }) + const paths = files.map(f => relative(tmpPath, f.path)).sort() + + expect(paths).to.eql([".gitmodules", join("sub", initFile), join("sub", "x.txt")]) + }) + + it("should respect include filter when scanning a submodule", async () => { + const path = join(tmpPath, "sub", "x.foo") + await createFile(path) + + const files = await handler.getFiles({ path: tmpPath, log, include: ["sub/*.txt"] }) + const paths = files.map(f => relative(tmpPath, f.path)).sort() + + expect(paths).to.eql([join("sub", initFile)]) + }) + + it("should respect exclude filter when scanning a submodule", async () => { + const path = join(tmpPath, "sub", "x.foo") + await createFile(path) + + const files = await handler.getFiles({ path: tmpPath, log, exclude: ["sub/*.txt"] }) + const paths = files.map(f => relative(tmpPath, f.path)).sort() + + expect(paths).to.eql([".gitmodules", join("sub", "x.foo")]) + }) + + context("submodule contains another submodule", () => { + let submoduleB: tmp.DirectoryResult + let submodulePathB: string + let initFileB: string + + beforeEach(async () => { + submoduleB = await makeTempGitRepo() + submodulePathB = await realpath(submoduleB.path) + initFileB = await commit("init", submodulePathB) + + await execa("git", ["submodule", "add", submodulePathB, "sub-b"], { cwd: join(tmpPath, "sub") }) + await execa("git", ["commit", "-m", "add submodule"], { cwd: join(tmpPath, "sub") }) + }) + + afterEach(async () => { + await submoduleB.cleanup() + }) + + it("should include tracked files in nested submodules", async () => { + const files = await handler.getFiles({ path: tmpPath, log }) + const paths = files.map(f => relative(tmpPath, f.path)).sort() + + expect(paths).to.eql([ + ".gitmodules", + join("sub", ".gitmodules"), + join("sub", initFile), + join("sub", "sub-b", initFileB), + ]) + }) + + it("should include untracked files in nested submodules", async () => { + const dir = join(tmpPath, "sub", "sub-b") + const path = join(dir, "x.txt") + await createFile(path) + + const files = await handler.getFiles({ path: tmpPath, log }) + const paths = files.map(f => relative(tmpPath, f.path)).sort() + + expect(paths).to.eql([ + ".gitmodules", + join("sub", ".gitmodules"), + join("sub", initFile), + join("sub", "sub-b", initFileB), + join("sub", "sub-b", "x.txt"), + ]) + }) + }) + }) }) describe("hashObject", () => { @@ -375,12 +462,15 @@ describe("GitHandler", () => { let clonePath: string beforeEach(async () => { - tmpRepoA = await makeTempGitRepo("test commit A") + tmpRepoA = await makeTempGitRepo() tmpRepoPathA = await realpath(tmpRepoA.path) + await commit("test commit A", tmpRepoPathA) + repositoryUrlA = `file://${tmpRepoPathA}#master` - tmpRepoB = await makeTempGitRepo("test commit B") + tmpRepoB = await makeTempGitRepo() tmpRepoPathB = await realpath(tmpRepoB.path) + await commit("test commit B", tmpRepoPathB) const hash = hashRepoUrl(repositoryUrlA) clonePath = join(tmpPath, "sources", "module", `foo--${hash}`)