From d1aaf5e4187829d183576cca75f7748715c90f00 Mon Sep 17 00:00:00 2001 From: Thorarinn Sigurdsson Date: Thu, 24 May 2018 03:17:14 +0200 Subject: [PATCH] fix: #107 & #108 - incl. deps in auto-reload. When specifying which modules to watch, modules for their build & service dependencies are now also watched. Additionally, service dependencies are now included as auto reload dependants. --- src/plugin-context.ts | 6 ++- src/types/module.ts | 10 +++++ src/watch.ts | 41 ++++++++++++++--- test/data/test-project-auto-reload/garden.yml | 6 +++ .../test-project-auto-reload}/module-a/foo | 0 .../module-a/garden.yml | 14 ++++++ .../test-project-auto-reload}/module-b/foo | 0 .../module-b/garden.yml | 8 +++- .../test-project-auto-reload}/module-c/foo | 0 .../module-c/garden.yml | 6 ++- .../test-project-auto-reload}/module-d/foo | 0 .../module-d/garden.yml | 6 ++- .../test-project-auto-reload}/module-e/foo | 0 .../module-e/garden.yml | 8 ++-- .../test-project-auto-reload}/module-f/foo | 0 .../module-f/garden.yml | 6 ++- .../data/auto-reload-project/garden.yml | 7 --- .../auto-reload-project/module-a/garden.yml | 5 --- test/src/watch.ts | 44 +++++++++++++++++++ 19 files changed, 140 insertions(+), 27 deletions(-) create mode 100644 test/data/test-project-auto-reload/garden.yml rename test/{src/commands/data/auto-reload-project => data/test-project-auto-reload}/module-a/foo (100%) create mode 100644 test/data/test-project-auto-reload/module-a/garden.yml rename test/{src/commands/data/auto-reload-project => data/test-project-auto-reload}/module-b/foo (100%) rename test/{src/commands/data/auto-reload-project => data/test-project-auto-reload}/module-b/garden.yml (60%) rename test/{src/commands/data/auto-reload-project => data/test-project-auto-reload}/module-c/foo (100%) rename test/{src/commands/data/auto-reload-project => data/test-project-auto-reload}/module-c/garden.yml (64%) rename test/{src/commands/data/auto-reload-project => data/test-project-auto-reload}/module-d/foo (100%) rename test/{src/commands/data/auto-reload-project => data/test-project-auto-reload}/module-d/garden.yml (72%) rename test/{src/commands/data/auto-reload-project => data/test-project-auto-reload}/module-e/foo (100%) rename test/{src/commands/data/auto-reload-project => data/test-project-auto-reload}/module-e/garden.yml (72%) rename test/{src/commands/data/auto-reload-project => data/test-project-auto-reload}/module-f/foo (100%) rename test/{src/commands/data/auto-reload-project => data/test-project-auto-reload}/module-f/garden.yml (72%) delete mode 100644 test/src/commands/data/auto-reload-project/garden.yml delete mode 100644 test/src/commands/data/auto-reload-project/module-a/garden.yml create mode 100644 test/src/watch.ts diff --git a/src/plugin-context.ts b/src/plugin-context.ts index 86c2f1916a..2413ce9942 100644 --- a/src/plugin-context.ts +++ b/src/plugin-context.ts @@ -85,6 +85,7 @@ import { sleep, } from "./util" import { + autoReloadModules, computeAutoReloadDependants, FSWatcher, } from "./watch" @@ -479,7 +480,8 @@ export function createPluginContext(garden: Garden): PluginContext { return results } - const autoReloadDependants = await computeAutoReloadDependants(modules) + const modulesToWatch = await autoReloadModules(modules) + const autoReloadDependants = await computeAutoReloadDependants(ctx) async function handleChanges(module: Module) { const tasks = await process(module) @@ -498,7 +500,7 @@ export function createPluginContext(garden: Garden): PluginContext { const watcher = new FSWatcher(ctx.projectRoot) // TODO: should the prefix here be different or set explicitly per run? - await watcher.watchModules(modules, "addTasksForAutoReload/", + await watcher.watchModules(modulesToWatch, "addTasksForAutoReload/", async (changedModule) => { ctx.log.debug({ msg: `Files changed for module ${changedModule.name}` }) await handleChanges(changedModule) diff --git a/src/types/module.ts b/src/types/module.ts index 0661d24fe3..2666e1344c 100644 --- a/src/types/module.ts +++ b/src/types/module.ts @@ -13,8 +13,10 @@ import { } from "fs" import * as Joi from "joi" import { + flatten, keyBy, set, + uniq, } from "lodash" import { join } from "path" import { GARDEN_VERSIONFILE_NAME } from "../constants" @@ -257,6 +259,14 @@ export class Module< return this.ctx.getServices(serviceNames) } + async getServiceDependencies(): Promise { + const depNames: string[] = uniq(flatten(this.services + .map(serviceConfig => serviceConfig.dependencies) + .filter(deps => deps))) + + return this.ctx.getServices(depNames) + } + async getDeployTasks( { force = false, forceBuild = false }: { force?: boolean, forceBuild?: boolean }, ): Promise { diff --git a/src/watch.ts b/src/watch.ts index e98a66ca69..68673412e6 100644 --- a/src/watch.ts +++ b/src/watch.ts @@ -8,7 +8,7 @@ import { map as bluebirdMap } from "bluebird" import { Client } from "fb-watchman" -import { keyBy } from "lodash" +import { keyBy, uniqBy, values } from "lodash" import { relative, resolve } from "path" import { Module } from "./types/module" import { PluginContext } from "./plugin-context" @@ -19,13 +19,36 @@ export interface OnChangeHandler { (ctx: PluginContext, module: Module): Promise } -export async function computeAutoReloadDependants(modules: Module[]): +/* + Resolves to modules and their build & service dependency modules (recursively). + Each module is represented at most once in the output. +*/ +export async function autoReloadModules(modules: Module[]): Promise { + const modulesByName = {} + + const scanner = async (module: Module, byName: object) => { + byName[module.name] = module + for (const dep of await uniqueDependencyModules(module)) { + if (!byName[dep.name]) { + await scanner(dep, byName) + } + } + } + + for (const m of modules) { + await scanner(m, modulesByName) + } + + return values(modulesByName) +} + +export async function computeAutoReloadDependants(ctx: PluginContext): Promise { - let dependants = {} + const dependants = {} - for (const module of modules) { - const deps = await module.getBuildDependencies() - for (const dep of deps) { + for (const module of await ctx.getModules()) { + const depModules: Module[] = await uniqueDependencyModules(module) + for (const dep of depModules) { dependants[dep.name] = (dependants[dep.name] || new Set()).add(module) } } @@ -33,6 +56,12 @@ export async function computeAutoReloadDependants(modules: Module[]): return dependants } +async function uniqueDependencyModules(module: Module): Promise { + const buildDepModules = await module.getBuildDependencies() + const serviceDepModules = (await module.getServiceDependencies()).map(s => s.module) + return uniqBy(buildDepModules.concat(serviceDepModules), m => m.name) +} + export type CapabilityOptions = { required?: string[], optional?: string[] } export type CapabilityResponse = { error: Error, response: { capabilities: { string: boolean } } } diff --git a/test/data/test-project-auto-reload/garden.yml b/test/data/test-project-auto-reload/garden.yml new file mode 100644 index 0000000000..42864b23ab --- /dev/null +++ b/test/data/test-project-auto-reload/garden.yml @@ -0,0 +1,6 @@ +project: + name: test-project-auto-reload + environments: + - name: local + providers: + - name: test-plugin diff --git a/test/src/commands/data/auto-reload-project/module-a/foo b/test/data/test-project-auto-reload/module-a/foo similarity index 100% rename from test/src/commands/data/auto-reload-project/module-a/foo rename to test/data/test-project-auto-reload/module-a/foo diff --git a/test/data/test-project-auto-reload/module-a/garden.yml b/test/data/test-project-auto-reload/module-a/garden.yml new file mode 100644 index 0000000000..bf8d5b30ff --- /dev/null +++ b/test/data/test-project-auto-reload/module-a/garden.yml @@ -0,0 +1,14 @@ +module: + name: module-a + type: container + image: scratch + services: + - name: service-a + endpoints: + - paths: [/path-b] + port: http + ports: + - name: http + containerPort: 8080 + build: + command: echo A diff --git a/test/src/commands/data/auto-reload-project/module-b/foo b/test/data/test-project-auto-reload/module-b/foo similarity index 100% rename from test/src/commands/data/auto-reload-project/module-b/foo rename to test/data/test-project-auto-reload/module-b/foo diff --git a/test/src/commands/data/auto-reload-project/module-b/garden.yml b/test/data/test-project-auto-reload/module-b/garden.yml similarity index 60% rename from test/src/commands/data/auto-reload-project/module-b/garden.yml rename to test/data/test-project-auto-reload/module-b/garden.yml index 9d3fb9bcd5..e205f70995 100644 --- a/test/src/commands/data/auto-reload-project/module-b/garden.yml +++ b/test/data/test-project-auto-reload/module-b/garden.yml @@ -1,10 +1,16 @@ module: name: module-b - type: generic + type: container + image: scratch services: - name: service-b + dependencies: + - service-a endpoints: - paths: [/path-b] + port: http + ports: + - name: http containerPort: 8080 build: command: echo B diff --git a/test/src/commands/data/auto-reload-project/module-c/foo b/test/data/test-project-auto-reload/module-c/foo similarity index 100% rename from test/src/commands/data/auto-reload-project/module-c/foo rename to test/data/test-project-auto-reload/module-c/foo diff --git a/test/src/commands/data/auto-reload-project/module-c/garden.yml b/test/data/test-project-auto-reload/module-c/garden.yml similarity index 64% rename from test/src/commands/data/auto-reload-project/module-c/garden.yml rename to test/data/test-project-auto-reload/module-c/garden.yml index a4e081519e..b03a6a638e 100644 --- a/test/src/commands/data/auto-reload-project/module-c/garden.yml +++ b/test/data/test-project-auto-reload/module-c/garden.yml @@ -1,10 +1,14 @@ module: name: module-c - type: generic + type: container + image: scratch services: - name: service-c endpoints: - paths: [/path-c] + port: http + ports: + - name: http containerPort: 8080 build: command: echo C diff --git a/test/src/commands/data/auto-reload-project/module-d/foo b/test/data/test-project-auto-reload/module-d/foo similarity index 100% rename from test/src/commands/data/auto-reload-project/module-d/foo rename to test/data/test-project-auto-reload/module-d/foo diff --git a/test/src/commands/data/auto-reload-project/module-d/garden.yml b/test/data/test-project-auto-reload/module-d/garden.yml similarity index 72% rename from test/src/commands/data/auto-reload-project/module-d/garden.yml rename to test/data/test-project-auto-reload/module-d/garden.yml index d3ecd4902c..ac1467df15 100644 --- a/test/src/commands/data/auto-reload-project/module-d/garden.yml +++ b/test/data/test-project-auto-reload/module-d/garden.yml @@ -1,12 +1,16 @@ module: name: module-d - type: generic + type: container + image: scratch services: - name: service-d dependencies: - service-b endpoints: - paths: [/path-d] + port: http + ports: + - name: http containerPort: 8080 build: command: echo D diff --git a/test/src/commands/data/auto-reload-project/module-e/foo b/test/data/test-project-auto-reload/module-e/foo similarity index 100% rename from test/src/commands/data/auto-reload-project/module-e/foo rename to test/data/test-project-auto-reload/module-e/foo diff --git a/test/src/commands/data/auto-reload-project/module-e/garden.yml b/test/data/test-project-auto-reload/module-e/garden.yml similarity index 72% rename from test/src/commands/data/auto-reload-project/module-e/garden.yml rename to test/data/test-project-auto-reload/module-e/garden.yml index 484c94061f..c9f929aa56 100644 --- a/test/src/commands/data/auto-reload-project/module-e/garden.yml +++ b/test/data/test-project-auto-reload/module-e/garden.yml @@ -1,16 +1,18 @@ module: name: module-e - type: generic + type: container + image: scratch services: - name: service-e dependencies: - - service-b - service-c endpoints: - paths: [/path-e] + port: http + ports: + - name: http containerPort: 8080 build: dependencies: - module-b - - module-c command: echo E diff --git a/test/src/commands/data/auto-reload-project/module-f/foo b/test/data/test-project-auto-reload/module-f/foo similarity index 100% rename from test/src/commands/data/auto-reload-project/module-f/foo rename to test/data/test-project-auto-reload/module-f/foo diff --git a/test/src/commands/data/auto-reload-project/module-f/garden.yml b/test/data/test-project-auto-reload/module-f/garden.yml similarity index 72% rename from test/src/commands/data/auto-reload-project/module-f/garden.yml rename to test/data/test-project-auto-reload/module-f/garden.yml index 8a8aa784cd..4466746eda 100644 --- a/test/src/commands/data/auto-reload-project/module-f/garden.yml +++ b/test/data/test-project-auto-reload/module-f/garden.yml @@ -1,12 +1,16 @@ module: name: module-f - type: generic + type: container + image: scratch services: - name: service-f dependencies: - service-c endpoints: - paths: [/path-f] + port: http + ports: + - name: http containerPort: 8080 build: dependencies: diff --git a/test/src/commands/data/auto-reload-project/garden.yml b/test/src/commands/data/auto-reload-project/garden.yml deleted file mode 100644 index 35c2ec9b15..0000000000 --- a/test/src/commands/data/auto-reload-project/garden.yml +++ /dev/null @@ -1,7 +0,0 @@ -project: - name: auto-reload-project - environments: - - name: local - providers: - - name: test - type: test-plugin diff --git a/test/src/commands/data/auto-reload-project/module-a/garden.yml b/test/src/commands/data/auto-reload-project/module-a/garden.yml deleted file mode 100644 index 0ec39c075a..0000000000 --- a/test/src/commands/data/auto-reload-project/module-a/garden.yml +++ /dev/null @@ -1,5 +0,0 @@ -module: - name: module-a - type: generic - build: - command: echo A diff --git a/test/src/watch.ts b/test/src/watch.ts new file mode 100644 index 0000000000..e89fcd21c5 --- /dev/null +++ b/test/src/watch.ts @@ -0,0 +1,44 @@ +import { expect } from "chai" +import { flatten, keys, mapValues, values, uniq } from "lodash" +import { join } from "path" +import { + AutoReloadDependants, + autoReloadModules, + computeAutoReloadDependants, +} from "../../src/watch" +import { makeTestGarden } from "../helpers" +import { inspect } from "util" + +const projectRoot = join(__dirname, "..", "data", "test-project-auto-reload") + +export function dependantModuleNames(ard: AutoReloadDependants): { [key: string]: string[] } { + return mapValues(ard, dependants => { + return Array.from(dependants).map(d => d.name) + }) +} + +describe("watch", () => { + describe("autoReloadModules", () => { + it("should include build and service dependencies of requested modules", async () => { + const ctx = (await makeTestGarden(projectRoot)).pluginContext + const moduleNames = (await autoReloadModules(await ctx.getModules(["module-e", "module-d"]))) + .map(m => m.name).sort() + + expect(moduleNames).to.eql(["module-a", "module-b", "module-c", "module-d", "module-e"]) + }) + }) + + describe("computeAutoReloadDependants", () => { + it("should include build and service dependants of requested modules", async () => { + const ctx = (await makeTestGarden(projectRoot)).pluginContext + const dependants = dependantModuleNames( + await computeAutoReloadDependants(ctx)) + + expect(dependants).to.eql({ + "module-a": [ "module-b" ], + "module-b": [ "module-d", "module-e" ], + "module-c": [ "module-e", "module-f" ], + }) + }) + }) +})