refactor: use events for file watching instead of callbacks

I started this as part of another branch, but figured I'd split this
into a new PR. This should have no functional differences to the prior
implementation, apart from exposing events that can be used in more
places. Added tests for the Watcher class along the way.

.circleci/config.yml

@@ -248,6 +248,8 @@ jobs:
           name: test
           command: |
             npm run ci-test
+          environment:
+            CHOKIDAR_USEPOLLING: "1"
   release-service-npm:
     <<: *node-config
     steps:

garden-service/package.json

@@ -117,12 +117,14 @@
     "@types/node": "^10.12.15",
     "@types/node-emoji": "^1.8.0",
     "@types/normalize-url": "^3.3.0",
+    "@types/p-event": "^1.3.0",
     "@types/p-queue": "^3.0.0",
     "@types/path-is-inside": "^1.0.0",
     "@types/prettyjson": "0.0.28",
     "@types/string-width": "^2.0.0",
     "@types/supertest": "^2.0.7",
     "@types/tar": "^4.0.0",
+    "@types/touch": "^3.1.1",
     "@types/uniqid": "^4.1.2",
     "@types/unzip": "^0.1.1",
     "@types/unzipper": "^0.9.1",
@@ -140,13 +142,15 @@
     "nock": "^10.0.4",
     "nodetree": "0.0.3",
     "nyc": "^13.1.0",
+    "p-event": "^2.1.0",
     "pegjs": "^0.10.0",
     "shx": "^0.3.2",
     "snyk": "^1.117.2",
     "testdouble": "^3.9.1",
     "testdouble-chai": "^0.5.0",
     "timekeeper": "^2.1.2",
     "tmp-promise": "^1.0.5",
+    "touch": "^3.1.0",
     "ts-node": "^7.0.1",
     "tslint": "^5.12.0",
     "tslint-microsoft-contrib": "^6.0.0",

garden-service/src/cli/cli.ts

@@ -296,6 +296,9 @@ export class GardenCli {
           args: parsedArgs,
           opts: parsedOpts,
         })
+
+        await garden.close()
+
       } while (result.restartRequired)
 
       // We attach the action result to cli context so that we can process it in the parse method

garden-service/src/events.ts

@@ -48,7 +48,27 @@ export class EventBus extends EventEmitter2 {
  * The supported events and their interfaces.
  */
 export interface Events {
+  // Internal test/control events
+  _restart: string,
   _test: string,
+
+  // Watcher events
+  configAdded: {
+    path: string,
+  },
+  projectConfigChanged: {},
+  moduleConfigChanged: {
+    name: string,
+  },
+  moduleSourcesChanged: {
+    name: string,
+    pathChanged: string,
+  },
+  moduleRemoved: {
+    name: string,
+  },
+
+  // TaskGraph events
   taskPending: {
     addedAt: Date,
     key: string,

garden-service/src/garden.ts

@@ -52,6 +52,7 @@ import {
   pickKeys,
   throwOnMissingNames,
   uniqByName,
+  Ignorer,
 } from "./util/util"
 import {
   DEFAULT_NAMESPACE,
@@ -108,6 +109,7 @@ import { SUPPORTED_PLATFORMS, SupportedPlatform } from "./constants"
 import { platform, arch } from "os"
 import { LogEntry } from "./logger/log-entry"
 import { EventBus } from "./events"
+import { Watcher } from "./watch"
 
 export interface ActionHandlerMap<T extends keyof PluginActions> {
   [actionName: string]: PluginActions[T]
@@ -154,6 +156,7 @@ export class Garden {
   private readonly taskNameIndex: { [key: string]: string } // task name -> module name
   private readonly hotReloadScheduler: HotReloadScheduler
   private readonly taskGraph: TaskGraph
+  private readonly watcher: Watcher
 
   public readonly environment: Environment
   public readonly localConfigStore: LocalConfigStore
@@ -169,6 +172,7 @@ export class Garden {
     variables: PrimitiveMap,
     public readonly projectSources: SourceConfig[] = [],
     public readonly buildDir: BuildDir,
+    public readonly ignorer: Ignorer,
     public readonly opts: GardenOpts,
   ) {
     // make sure we're on a supported platform
@@ -209,6 +213,7 @@ export class Garden {
     this.actions = new ActionHelper(this)
     this.hotReloadScheduler = new HotReloadScheduler()
     this.events = new EventBus()
+    this.watcher = new Watcher(this, this.log)
   }
 
   static async factory<T extends typeof Garden>(
@@ -293,6 +298,7 @@ export class Garden {
     const variables = merge({}, environmentDefaults.variables, environmentConfig.variables)
 
     const buildDir = await BuildDir.factory(projectRoot)
+    const ignorer = await getIgnorer(projectRoot)
 
     const garden = new this(
       projectRoot,
@@ -301,6 +307,7 @@ export class Garden {
       variables,
       projectSources,
       buildDir,
+      ignorer,
       opts,
     ) as InstanceType<T>
 
@@ -319,6 +326,13 @@ export class Garden {
     return garden
   }
 
+  /**
+   * Clean up before shutting down.
+   */
+  async close() {
+    this.watcher.stop()
+  }
+
   getPluginContext(providerName: string) {
     return createPluginContext(this, providerName)
   }
@@ -339,6 +353,15 @@ export class Garden {
     return this.hotReloadScheduler.requestHotReload(moduleName, hotReloadHandler)
   }
 
+  /**
+   * Enables the file watcher for the project.
+   * Make sure to stop it using `.close()` when cleaning up or when watching is no longer needed.
+   */
+  async startWatcher() {
+    const modules = await this.getModules()
+    this.watcher.start(modules)
+  }
+
   private registerPlugin(name: string, moduleOrFactory: RegisterPluginParam) {
     let factory: PluginFactory
 

garden-service/src/process.ts

@@ -8,14 +8,12 @@
 
 import Bluebird = require("bluebird")
 import chalk from "chalk"
-import { padEnd } from "lodash"
+import { padEnd, keyBy } from "lodash"
 
 import { Module } from "./types/module"
 import { Service } from "./types/service"
 import { BaseTask } from "./tasks/base"
 import { TaskResults } from "./task-graph"
-import { FSWatcher } from "./watch"
-import { registerCleanupFunction } from "./util/util"
 import { isModuleLinked } from "./util/ext-source-util"
 import { Garden } from "./garden"
 import { LogEntry } from "./logger/log-entry"
@@ -110,32 +108,40 @@ export async function processModules(
     changeHandler = handler
   }
 
-  const watcher = new FSWatcher(garden, log)
-
-  const restartPromise = new Promise(async (resolve) => {
-    await watcher.watchModules(modules,
-      async (changedModule: Module | null, configChanged: boolean) => {
-        if (configChanged) {
-          if (changedModule) {
-            log.info({ section: changedModule.name, msg: `Module configuration changed, reloading...`, symbol: "info" })
-          } else {
-            log.info({ symbol: "info", msg: `Project configuration changed, reloading...` })
-          }
-          resolve()
-          return
-        }
-
-        if (changedModule) {
-          log.silly({ msg: `Files changed for module ${changedModule.name}` })
-          changedModule = await garden.getModule(changedModule.name)
-          await Bluebird.map(changeHandler!(changedModule), (task) => garden.addTask(task))
-        }
-
-        await garden.processTasks()
-      })
-
-    registerCleanupFunction("clearAutoReloadWatches", () => {
-      watcher.close()
+  const modulesByName = keyBy(modules, "name")
+
+  await garden.startWatcher()
+
+  const restartPromise = new Promise((resolve) => {
+    garden.events.on("_restart", () => {
+      log.debug({ symbol: "info", msg: `Manual restart triggered` })
+      resolve()
+    })
+
+    garden.events.on("projectConfigChanged", () => {
+      log.info({ symbol: "info", msg: `Project configuration changed, reloading...` })
+      resolve()
+    })
+
+    garden.events.on("configAdded", (event) => {
+      log.info({ symbol: "info", msg: `Garden config added at ${event.path}, reloading...` })
+      resolve()
+    })
+
+    garden.events.on("moduleConfigChanged", (event) => {
+      log.info({ symbol: "info", section: event.name, msg: `Module configuration changed, reloading...` })
+      resolve()
+    })
+
+    garden.events.on("moduleSourcesChanged", async (event) => {
+      const changedModule = modulesByName[event.name]
+
+      if (!changedModule) {
+        return
+      }
+
+      await Bluebird.map(changeHandler!(changedModule), (task) => garden.addTask(task))
+      await garden.processTasks()
     })
   })
 
@@ -147,7 +153,6 @@ export async function processModules(
   }
 
   await restartPromise
-  watcher.close()
 
   return {
     taskResults: {}, // TODO: Return latest results for each task baseKey processed between restarts?

garden-service/src/util/util.ts

@@ -109,7 +109,11 @@ export async function getChildDirNames(parentDir: string): Promise<string[]> {
   return dirNames
 }
 
-export async function getIgnorer(rootPath: string) {
+export interface Ignorer {
+  ignores: (path: string) => boolean
+}
+
+export async function getIgnorer(rootPath: string): Promise<Ignorer> {
   // TODO: this doesn't handle nested .gitignore files, we should revisit
   const gitignorePath = join(rootPath, ".gitignore")
   const gardenignorePath = join(rootPath, ".gardenignore")