Merge pull request #1071 from garden-io/remote-sources-fix-watch

Remote sources fix watch

dashboard/src/components/graph/index.tsx

@@ -135,7 +135,7 @@ function drawChart(
   const svgGroup = svg.append("g")
 
   // Set up zoom support
-  const zoom = d3.zoom<SVGSVGElement, {}>().on("zoom", () => {
+  const zoom = d3.zoom<SVGSVGElement, any>().on("zoom", () => {
     svgGroup.attr("transform", d3.event.transform)
   })
   svg.call(zoom)

garden-service/src/commands/unlink/module.ts

@@ -71,7 +71,7 @@ export class UnlinkModuleCommand extends Command<Args, Opts> {
 
     const linkedModuleSources = await removeLinkedSources({ garden, sourceType, names: modules })
 
-    log.info(`Unlinked module(s) ${module}`)
+    log.info(`Unlinked module(s) ${modules.join(" ")}`)
 
     return { result: linkedModuleSources }
   }

garden-service/src/commands/unlink/source.ts

@@ -71,7 +71,7 @@ export class UnlinkSourceCommand extends Command<Args, Opts> {
 
     const linkedProjectSources = await removeLinkedSources({ garden, sourceType, names: sources })
 
-    log.info(`Unlinked source(s) ${sources}`)
+    log.info(`Unlinked source(s) ${sources.join(" ")}`)
 
     return { result: linkedProjectSources }
   }

garden-service/src/garden.ts

@@ -220,6 +220,10 @@ export class Garden {
     const buildDir = await BuildDir.factory(projectRoot, gardenDirPath)
     const workingCopyId = await getWorkingCopyId(gardenDirPath)
 
+    // We always exclude the garden dir
+    const gardenDirExcludePattern = `${relative(projectRoot, gardenDirPath)}/**/*`
+    const moduleExcludePatterns = [...((config.modules || {}).exclude || []), gardenDirExcludePattern]
+
     const garden = new this({
       projectRoot,
       projectName,
@@ -231,10 +235,10 @@ export class Garden {
       opts,
       plugins,
       providerConfigs: providers,
+      moduleExcludePatterns,
       workingCopyId,
       dotIgnoreFiles: config.dotIgnoreFiles,
       moduleIncludePatterns: (config.modules || {}).include,
-      moduleExcludePatterns: (config.modules || {}).exclude,
     }) as InstanceType<T>
 
     return garden
@@ -265,7 +269,9 @@ export class Garden {
    */
   async startWatcher(graph: ConfigGraph) {
     const modules = await graph.getModules()
-    this.watcher = new Watcher(this, this.log, modules)
+    const linkedPaths = (await getLinkedSources(this)).map(s => s.path)
+    const paths = [this.projectRoot, ...linkedPaths]
+    this.watcher = new Watcher(this, this.log, paths, modules)
   }
 
   private registerPlugin(name: string, moduleOrFactory: RegisterPluginParam) {

garden-service/src/util/ext-source-util.ts

@@ -60,14 +60,21 @@ export function isModuleLinked(module: Module, garden: Garden) {
   return !pathIsInside(module.path, garden.projectRoot) && !isPluginModule
 }
 
-export async function getLinkedSources(
-  garden: Garden,
-  type: ExternalSourceType,
-): Promise<LinkedSource[]> {
+/**
+ * Returns an array of linked sources by type, as read from the local config store.
+ * Returns all linked sources if typed not specified.
+ */
+export async function getLinkedSources(garden: Garden, type?: ExternalSourceType): Promise<LinkedSource[]> {
   const localConfig = await garden.configStore.get()
-  return (type === "project"
-    ? localConfig.linkedProjectSources
-    : localConfig.linkedModuleSources) || []
+  const linkedModuleSources = localConfig.linkedModuleSources || []
+  const linkedProjectSources = localConfig.linkedProjectSources || []
+  if (type === "module") {
+    return linkedModuleSources
+  } else if (type === "project") {
+    return linkedProjectSources
+  } else {
+    return [...linkedModuleSources, ...linkedProjectSources]
+  }
 }
 
 export async function addLinkedSources({ garden, sourceType, sources }: {

garden-service/src/util/fs.ts

@@ -21,7 +21,7 @@ import { VcsHandler } from "../vcs/vcs"
 const VALID_CONFIG_FILENAMES = ["garden.yml", "garden.yaml"]
 const metadataFilename = "metadata.json"
 export const defaultDotIgnoreFiles = [".gitignore", ".gardenignore"]
-export const fixedExcludes = [".git", ".garden", "debug-info*/**"]
+export const fixedExcludes = [".git", ".garden/**/*", "debug-info*/**"]
 
 /*
   Warning: Don't make any async calls in the loop body when using this function, since this may cause

garden-service/src/vcs/git.ts

@@ -87,13 +87,15 @@ export class GitHandler extends VcsHandler {
        * We need to exclude .garden to avoid errors when path is the project root. This happens e.g. for modules
        * whose config is colocated with the project config, and that don't specify include paths/patterns.
        */
-      lines = await git("ls-files", "-s", "--other", "--exclude=.garden", path)
+      // FIXME: We should use `garden.gardenDirPath` instead of ".garden" since the gardenDirPath
+      // property is configurable.
+      lines = await git("ls-files", "-s", "--others", "--exclude=.garden", path)
 
       // List ignored files from .gardenignore. We need to run ls-files twice to get both tracked and untracked files.
-      const lsFilesCmd = ["ls-files", "--ignored", ...this.ignoreFiles.map(f => `--exclude-per-directory=${f}`)]
-      const lsFilesUntrackedCmd = [...lsFilesCmd, "--others"]
+      const lsIgnoredFiles = ["ls-files", "--ignored", ...this.ignoreFiles.map(f => `--exclude-per-directory=${f}`)]
+      const lsUntrackedIgnoredFiles = [...lsIgnoredFiles, "--others"]
 
-      ignored = flatten(await Bluebird.map([lsFilesCmd, lsFilesUntrackedCmd], async (cmd) => git(...cmd, path)))
+      ignored = flatten(await Bluebird.map([lsIgnoredFiles, lsUntrackedIgnoredFiles], async (cmd) => git(...cmd, path)))
     } catch (err) {
       // if we get 128 we're not in a repo root, so we get no files
       if (err.code !== 128) {

garden-service/src/watch.ts

@@ -20,15 +20,17 @@ import { isConfigFilename } from "./util/fs"
 // IMPORTANT: We must use a single global instance of the watcher, because we may otherwise get
 // segmentation faults on macOS! See https://github.com/fsevents/fsevents/issues/273
 let watcher: FSWatcher | undefined
-let projectRoot: string
 
-// The process hangs after tests if we don't do this
-registerCleanupFunction("stop watcher", () => {
+// Export so that we can clean up the global watcher instance when running tests
+export function cleanUpGlobalWatcher() {
   if (watcher) {
     watcher.close()
     watcher = undefined
   }
-})
+}
+
+// The process hangs after tests if we don't do this
+registerCleanupFunction("stop watcher", cleanUpGlobalWatcher)
 
 export type ChangeHandler = (module: Module | null, configChanged: boolean) => Promise<void>
 
@@ -39,13 +41,11 @@ export type ChangeHandler = (module: Module | null, configChanged: boolean) => P
 export class Watcher {
   private watcher: FSWatcher
 
-  constructor(private garden: Garden, private log: LogEntry, modules: Module[]) {
-    projectRoot = this.garden.projectRoot
-
-    this.log.debug(`Watcher: Watching ${projectRoot}`)
+  constructor(private garden: Garden, private log: LogEntry, paths: string[], modules: Module[]) {
+    log.debug(`Watcher: Watching paths ${paths.join(", ")}`)
 
     if (watcher === undefined) {
-      watcher = watch(projectRoot, {
+      watcher = watch(paths, {
         ignoreInitial: true,
         persistent: true,
       })

garden-service/test/helpers.ts

@@ -7,9 +7,12 @@
  */
 
 import * as td from "testdouble"
+import Bluebird = require("bluebird")
 import { resolve, join } from "path"
 import { extend } from "lodash"
-import { remove, readdirSync, existsSync } from "fs-extra"
+import { remove, readdirSync, existsSync, copy, mkdirp, pathExists, truncate } from "fs-extra"
+import execa = require("execa")
+
 import { containerModuleSpecSchema, containerTestSchema, containerTaskSchema } from "../src/plugins/container/config"
 import { testExecModule, buildExecModule, execBuildSpecSchema } from "../src/plugins/exec"
 import { TaskResults } from "../src/task-graph"
@@ -25,7 +28,7 @@ import { Garden, GardenParams } from "../src/garden"
 import { ModuleConfig } from "../src/config/module"
 import { mapValues, fromPairs } from "lodash"
 import { ModuleVersion } from "../src/vcs/vcs"
-import { GARDEN_SERVICE_ROOT } from "../src/constants"
+import { GARDEN_SERVICE_ROOT, LOCAL_CONFIG_FILENAME } from "../src/constants"
 import { EventBus, Events } from "../src/events"
 import { ValueOf } from "../src/util/util"
 import { LogEntry } from "../src/logger/log-entry"
@@ -39,6 +42,7 @@ import { DeleteSecretParams } from "../src/types/plugin/provider/deleteSecret"
 import { RunServiceParams } from "../src/types/plugin/service/runService"
 import { RunTaskParams, RunTaskResult } from "../src/types/plugin/task/runTask"
 import { RunResult } from "../src/types/plugin/base"
+import { ExternalSourceType, getRemoteSourceRelPath, hashRepoUrl } from "../src/util/ext-source-util"
 
 export const dataDir = resolve(GARDEN_SERVICE_ROOT, "test", "unit", "data")
 export const examplesDir = resolve(GARDEN_SERVICE_ROOT, "..", "examples")
@@ -50,6 +54,10 @@ export const testModuleVersion: ModuleVersion = {
   files: [],
 }
 
+// All test projects use this git URL
+export const testGitUrl = "https://my-git-server.com/my-repo.git#master"
+export const testGitUrlHash = hashRepoUrl(testGitUrl)
+
 export function getDataDir(...names: string[]) {
   return resolve(dataDir, ...names)
 }
@@ -379,20 +387,6 @@ export const cleanProject = async (gardenDirPath: string) => {
   return remove(gardenDirPath)
 }
 
-/**
- * Prevents git cloning. Use if creating a Garden instance with test-project-ext-module-sources
- * or test-project-ext-project-sources as project root.
- */
-export function stubExtSources(garden: Garden) {
-  td.replace(garden.vcs, "cloneRemoteSource", async () => undefined)
-  td.replace(garden.vcs, "updateRemoteSource", async () => undefined)
-
-  const getRemoteSourcesDirname = td.replace(garden.vcs, "getRemoteSourcesDirname")
-
-  td.when(getRemoteSourcesDirname("module")).thenReturn(join("sources", "module"))
-  td.when(getRemoteSourcesDirname("project")).thenReturn(join("sources", "project"))
-}
-
 export function getExampleProjects() {
   const names = readdirSync(examplesDir).filter(n => {
     const basePath = join(examplesDir, n)
@@ -416,3 +410,56 @@ export function freezeTime(date?: Date) {
   timekeeper.freeze(date)
   return date
 }
+
+export async function resetLocalConfig(gardenDirPath: string) {
+  const path = join(gardenDirPath, LOCAL_CONFIG_FILENAME)
+  if (await pathExists(path)) {
+    await truncate(path)
+  }
+}
+
+/**
+ * Idempotently initializes the test-project-ext-project-sources project and returns
+ * the Garden class.
+ */
+export async function makeExtProjectSourcesGarden() {
+  const projectRoot = resolve(dataDir, "test-project-ext-project-sources")
+  // Borrow the external sources from here:
+  const extSourcesRoot = resolve(dataDir, "test-project-local-project-sources")
+  const sourceNames = ["source-a", "source-b", "source-c"]
+  return prepareRemoteGarden({ projectRoot, extSourcesRoot, sourceNames, type: "project" })
+}
+
+/**
+ * Idempotently initializes the test-project-ext-project-sources project and returns
+ * the Garden class.
+ */
+export async function makeExtModuleSourcesGarden() {
+  const projectRoot = resolve(dataDir, "test-project-ext-module-sources")
+  // Borrow the external sources from here:
+  const extSourcesRoot = resolve(dataDir, "test-project-local-module-sources")
+  const sourceNames = ["module-a", "module-b", "module-c"]
+  return prepareRemoteGarden({ projectRoot, extSourcesRoot, sourceNames, type: "module" })
+}
+
+/**
+ * Helper function for idempotently initializing the ext-sources projects.
+ * Copies the external sources into the .garden directory and git inits them.
+ */
+async function prepareRemoteGarden({
+  projectRoot, extSourcesRoot, sourceNames, type,
+}: { projectRoot: string, extSourcesRoot: string, sourceNames: string[], type: ExternalSourceType }) {
+  const garden = await makeTestGarden(projectRoot)
+  const sourcesPath = join(projectRoot, ".garden", "sources", type)
+
+  await mkdirp(sourcesPath)
+  // Copy the sources to the `.garden/sources` dir and git init them
+  await Bluebird.map(sourceNames, async (name) => {
+    const remoteSourceRelPath = getRemoteSourceRelPath({ name, url: testGitUrl, sourceType: type })
+    const targetPath = join(projectRoot, ".garden", remoteSourceRelPath)
+    await copy(join(extSourcesRoot, name), targetPath)
+    await execa("git", ["init"], { cwd: targetPath })
+  })
+
+  return garden
+}

garden-service/test/unit/data/test-project-ext-module-sources/garden.yml

@@ -1,11 +1,6 @@
 project:
   name: test-project-ext-project-sources
-  environmentDefaults:
-    variables:
-      some: variable
   environments:
     - name: local
       providers:
         - name: test-plugin
-        - name: test-plugin-b
-    - name: other

garden-service/test/unit/data/test-project-ext-module-sources/module-a/garden.yml

@@ -1,6 +1,6 @@
 module:
   name: module-a
-  repositoryUrl: https://my-git-server.com/module-a.git#master
+  repositoryUrl: https://my-git-server.com/my-repo.git#master
   type: test
   services:
     - name: service-a

garden-service/test/unit/data/test-project-ext-module-sources/module-b/garden.yml

@@ -1,6 +1,6 @@
 module:
   name: module-b
-  repositoryUrl: https://my-git-server.com/module-b.git#master
+  repositoryUrl: https://my-git-server.com/my-repo.git#master
   type: test
   services:
     - name: service-b

garden-service/test/unit/data/test-project-ext-module-sources/module-c/garden.yml

@@ -1,6 +1,6 @@
 module:
   name: module-c
-  repositoryUrl: https://my-git-server.com/module-c.git#master
+  repositoryUrl: https://my-git-server.com/my-repo.git#master
   type: test
   services:
     - name: service-c

garden-service/test/unit/data/test-project-ext-project-sources/garden.yml

@@ -2,11 +2,11 @@ project:
   name: test-project-ext-project-sources
   sources:
     - name: source-a
-      repositoryUrl: https://my-git-server.com/source-a.git#master
+      repositoryUrl: https://my-git-server.com/my-repo.git#master
     - name: source-b
-      repositoryUrl: https://my-git-server.com/source-b.git#master
+      repositoryUrl: https://my-git-server.com/my-repo.git#master
     - name: source-c
-      repositoryUrl: https://my-git-server.com/source-c.git#master
+      repositoryUrl: https://my-git-server.com/my-repo.git#master
   environmentDefaults:
     variables:
       some: variable