fix(watcher): use exclude options to optimize file watching

This basically relies on optimizations already implemented in chokidar,
the library we use for file watching. Hopefully this can solve the
excessive CPU/RAM usage some users have reported.

Closes #1269

docs/guides/configuration-files.md

@@ -353,6 +353,8 @@ Here we only scan the `modules` directory, but exclude the `modules/tmp` directo
 
 If you specify a list with `include`, only those patterns are included. If you then specify one or more `exclude` patterns, those are filtered out of the ones matched by `include`. If you _only_ specify `exclude`, those patterns will be filtered out of all paths in the project directory.
 
+The `modules.exclude` field is also used to limit the number of files and directories Garden watches for changes while running. Use that if you have a large number of files/directories in your project that you do not need to watch, or if you are seeing excessive CPU/RAM usage.
+
 #### Module include/exclude
 
 By default, all files in the same directory as a module configuration file are included as source files for that module. Sometimes you need more granular control over the context, not least if you have multiple modules in the same directory.
@@ -375,6 +377,8 @@ Here we only include the `Dockerfile` and all the `.py` files under `my-sources/
 
 If you specify a list with `include`, only those files/patterns are included. If you then specify one or more `exclude` files or patterns, those are filtered out of the files matched by `include`. If you _only_ specify `exclude`, those patterns will be filtered out of all files in the module directory.
 
+The `exclude` field on modules is also used to limit the number of files and directories Garden watches for changes while running. Use that if you have a large number of files/directories in your module that you do not need to watch, or if you are seeing excessive CPU/RAM usage.
+
 #### .ignore files
 
 By default, Garden respects `.gitignore` and `.gardenignore` files and excludes any patterns matched in those files.

docs/troubleshooting.md

@@ -7,6 +7,12 @@ title: Troubleshooting
 
 _This section could (obviously) use more work. Contributions are most appreciated!_
 
+## I have a huge number of files in my repository and Garden is eating all my CPU/RAM
+
+This issue often comes up on Linux, and in other scenarios where the filesystem doesn't support event-based file watching.
+
+Thankfully, you can in most cases avoid this problem using the `modules.exclude` field in your project config, and/or the `exclude` field in your individual module configs. See the [Including/excluding files and directories](./using-garden/configuration-files#including-excluding-files-and-directories) section in our Configuration Files guide for details.
+
 ## I'm getting an "EPERM: operation not permitted, rename..." error on Windows
 
 This is a known issue with Windows and may affect many Node.js applications (and possibly others).

garden-service/package.json

@@ -36,7 +36,7 @@
     "certpem": "^1.1.3",
     "chalk": "^2.4.2",
     "child-process-promise": "^2.2.1",
-    "chokidar": "^3.0.2",
+    "chokidar": "^3.3.0",
     "ci-info": "^2.0.0",
     "circular-json": "^0.5.9",
     "cli-cursor": "^3.1.0",

garden-service/src/watch.ts

@@ -7,35 +7,20 @@
  */
 
 import { watch, FSWatcher } from "chokidar"
-import { parse, basename } from "path"
+import { parse, basename, resolve } from "path"
 import { pathToCacheContext } from "./cache"
 import { Module } from "./types/module"
 import { Garden } from "./garden"
 import { LogEntry } from "./logger/log-entry"
-import { registerCleanupFunction, sleep } from "./util/util"
-import { some } from "lodash"
+import { sleep } from "./util/util"
+import { some, flatten } from "lodash"
 import { isConfigFilename, matchPath } from "./util/fs"
 import Bluebird from "bluebird"
 import { InternalError } from "./exceptions"
 
 // How long we wait between processing added files and directories
 const DEFAULT_BUFFER_INTERVAL = 500
 
-// IMPORTANT: We must use a single global instance of the watcher, because we may otherwise get
-// segmentation faults on macOS! See https://github.com/fsevents/fsevents/issues/273
-let watcher: FSWatcher | undefined
-
-// Export so that we can clean up the global watcher instance when running tests
-export function cleanUpGlobalWatcher() {
-  if (watcher) {
-    watcher.close()
-    watcher = undefined
-  }
-}
-
-// The process hangs after tests if we don't do this
-registerCleanupFunction("stop watcher", cleanUpGlobalWatcher)
-
 export type ChangeHandler = (module: Module | null, configChanged: boolean) => Promise<void>
 
 type ChangeType = "added" | "changed" | "removed"
@@ -82,7 +67,9 @@ export class Watcher {
   start() {
     this.log.debug(`Watcher: Watching paths ${this.paths.join(", ")}`)
 
-    if (watcher === undefined) {
+    this.running = true
+
+    if (!this.watcher) {
       // Make sure that fsevents works when we're on macOS. This has come up before without us noticing, which has
       // a dramatic performance impact, so it's best if we simply throw here so that our tests catch such issues.
       if (process.platform === "darwin") {
@@ -95,21 +82,23 @@ export class Watcher {
         }
       }
 
-      watcher = watch(this.paths, {
+      // Collect all the configured excludes and pass to the watcher.
+      // This allows chokidar to optimize polling based on the exclusions.
+      // See https://github.com/garden-io/garden/issues/1269.
+      // TODO: see if we can extract paths from dotignore files as well (we'd have to deal with negations etc. somehow).
+      const projectExcludes = this.garden.moduleExcludePatterns.map((p) => resolve(this.garden.projectRoot, p))
+      const moduleExcludes = flatten(this.modules.map((m) => (m.exclude || []).map((p) => resolve(m.path, p))))
+
+      this.watcher = watch(this.paths, {
         ignoreInitial: true,
         ignorePermissionErrors: true,
         persistent: true,
         awaitWriteFinish: {
           stabilityThreshold: 500,
           pollInterval: 200,
         },
+        ignored: [...projectExcludes, ...moduleExcludes],
       })
-    }
-
-    this.running = true
-
-    if (!this.watcher) {
-      this.watcher = watcher
 
       this.watcher
         .on("add", this.makeFileAddedHandler())

garden-service/test/data/test-project-watch/.gardenignore

@@ -1 +1 @@
-**/project-excluded.txt
+**/gardenignore-excluded.txt

garden-service/test/data/test-project-watch/garden.yml

@@ -1,5 +1,7 @@
 kind: Project
 name: test-project-a
+modules:
+  exclude: [module-a/project-excluded.txt]
 environments:
   - name: local
     providers:

garden-service/test/data/test-project-watch/module-a/gardenignore-excluded.txt

@@ -0,0 +1 @@
+Nope!

garden-service/test/data/test-projects/huge-project/.gitignore

@@ -0,0 +1 @@
+dir*

garden-service/test/data/test-projects/huge-project/README.md

@@ -0,0 +1,16 @@
+# Huge project tests
+
+Created for [PR #1320](https://github.com/garden-io/garden/pull/1320).
+
+This one's a bit hard to automate, but we can use to make sure Garden can handle massive amounts of files in repositories.
+
+The procedure to test was as follows:
+
+1) `cd` to this directory.
+2) Run `node generate.js`.
+3) Comment out the `modules.exclude` field in the `garden.yml`.
+4) In `garden-service/src/watch.ts`, add `usePolling: true` to the options for the chokidar `watch()` function.
+5) Run `garden build -w` and observe the process drain CPU and RAM until it crashes in about a minute.
+6) Uncomment the `modules.exclude` field in the `garden.yml`.
+7) Run `garden build -w` again, wait and observe a happy process for a while.
+8) Run `rm -rf dir*` to clean up.

garden-service/test/data/test-projects/huge-project/garden.yml

@@ -0,0 +1,5 @@
+kind: Project
+name: huge-project
+dotIgnoreFiles: [.gardenignore]
+modules:
+  exclude: [dir0/**/*, dir1/**/*, dir2/**/*, dir3/**/*, dir4/**/*, dir5/**/*, dir6/**/*, dir7/**/*, dir8/**/*]

garden-service/test/data/test-projects/huge-project/generate.js

@@ -0,0 +1,42 @@
+const { join } = require("path")
+const { ensureDir, ensureFile } = require("fs-extra")
+
+let levels = 6
+let directoriesPerLevel = 6
+let filesPerLevel = 3
+
+async function generateData(cwd, level) {
+  level++
+
+  let files = 0
+  let directories = 0
+
+  for (let d = 0; d < directoriesPerLevel; d++) {
+    const dir = join(cwd, "dir" + d)
+    await ensureDir(dir)
+    directories++
+
+    for (let f = 0; f < filesPerLevel; f++) {
+      const file = join(dir, "file" + f)
+      await ensureFile(file)
+      files++
+    }
+
+    if (level < levels) {
+      const res = await generateData(dir, level)
+      files += res.files
+      directories += res.directories
+    }
+  }
+
+  return { files, directories }
+}
+
+generateData(process.cwd(), 0)
+  .then((res) => {
+    console.log(`Made ${res.files} files in ${res.directories} directories`)
+  })
+  .catch(err => {
+    console.log(err)
+    process.exit(1)
+  })

garden-service/test/unit/src/watch.ts

@@ -15,7 +15,6 @@ import { CacheContext, pathToCacheContext } from "../../../src/cache"
 import { createFile, remove, pathExists } from "fs-extra"
 import { getConfigFilePath } from "../../../src/util/fs"
 import { LinkModuleCommand } from "../../../src/commands/link/module"
-import { cleanUpGlobalWatcher } from "../../../src/watch"
 import { LinkSourceCommand } from "../../../src/commands/link/source"
 import { sleep } from "../../../src/util/util"
 
@@ -174,6 +173,12 @@ describe("Watcher", () => {
     })
   })
 
+  it("should not emit moduleSourcesChanged if file is changed and matches the modules.exclude list", async () => {
+    const pathChanged = resolve(includeModulePath, "project-excluded.txt")
+    emitEvent(garden, "change", pathChanged)
+    expect(garden.events.eventLog).to.eql([])
+  })
+
   it("should not emit moduleSourcesChanged if file is changed and doesn't match module's include list", async () => {
     const pathChanged = resolve(includeModulePath, "foo.txt")
     emitEvent(garden, "change", pathChanged)
@@ -187,7 +192,7 @@ describe("Watcher", () => {
   })
 
   it("should not emit moduleSourcesChanged if file is changed and it's in a gardenignore in the project", async () => {
-    const pathChanged = resolve(modulePath, "project-excluded.txt")
+    const pathChanged = resolve(modulePath, "gardenignore-excluded.txt")
     emitEvent(garden, "change", pathChanged)
     expect(garden.events.eventLog).to.eql([])
   })
@@ -281,8 +286,6 @@ describe("Watcher", () => {
     const localModulePathB = join(localModuleSourceDir, "module-b")
 
     before(async () => {
-      // The watcher instance is global so we clean up the previous one before proceeding
-      cleanUpGlobalWatcher()
       garden = await makeExtModuleSourcesGarden()
 
       // Link some modules
@@ -353,8 +356,6 @@ describe("Watcher", () => {
     const localSourcePathB = join(localProjectSourceDir, "source-b")
 
     before(async () => {
-      // The watcher instance is global so we clean up the previous one before proceeding
-      cleanUpGlobalWatcher()
       garden = await makeExtProjectSourcesGarden()
 
       // Link some projects