fix: module versions are now handled properly across multiple repos

This refactor resolves issues with using plugin modules as dependencies,
and is important for multi-repo support. Basically we now hash versions
together instead of using the latest commit, which would of course only
work within a single repo.

.gitignore

@@ -12,7 +12,6 @@ node_modules
 
 # Runtime files
 .garden
-.garden-version
 tmp/
 
 # TS cache on the CI
@@ -30,6 +29,7 @@ src/**/*.js
 src/**/*.map
 src/**/*.d.ts
 static/bin/garden.js
+static/**/.garden-version
 support/**/*.js
 support/**/*.map
 support/**/*.d.ts

src/plugin-context.ts

@@ -24,10 +24,8 @@ import {
   PrimitiveMap,
   validate,
 } from "./types/common"
-import {
-  Module,
-  versionFileSchema,
-} from "./types/module"
+import { Module } from "./types/module"
+import { moduleVersionSchema } from "./vcs/base"
 import {
   ModuleActions,
   Provider,
@@ -83,11 +81,15 @@ import {
   values,
   keyBy,
   omit,
+  flatten,
+  uniqBy,
+  sortBy,
 } from "lodash"
 import {
+  getNames,
   Omit,
 } from "./util/util"
-import { TreeVersion } from "./vcs/base"
+import { ModuleVersion } from "./vcs/base"
 
 export type PluginContextGuard = {
   readonly [P in keyof (PluginActionParams | ModuleActionParams<any>)]: (...args: any[]) => Promise<any>
@@ -162,8 +164,9 @@ export interface PluginContext extends PluginContextGuard, WrappedFromGarden {
   invalidateCacheUp: (context: CacheContext) => void
   invalidateCacheDown: (context: CacheContext) => void
   getModuleBuildPath: (moduleName: string) => Promise<string>
-  getModuleVersion: (moduleName: string, force?: boolean) => Promise<TreeVersion>
-  getLatestVersion: (versions: TreeVersion[]) => Promise<TreeVersion>
+  getModuleVersion: (moduleName: string, force?: boolean) => Promise<ModuleVersion>
+  resolveVersion: (moduleName: string, moduleDependencies: string[], force?: boolean) => Promise<ModuleVersion>
+  resolveModuleDependencies: (buildDependencies: string[], serviceDependencies: string[]) => Promise<Module[]>
   stageBuild: (moduleName: string) => Promise<void>
   getStatus: () => Promise<ContextStatus>
 }
@@ -433,30 +436,62 @@ export function createPluginContext(garden: Garden): PluginContext {
     },
 
     getModuleVersion: async (moduleName: string, force = false) => {
+      const dependencies = await ctx.resolveModuleDependencies([moduleName], [])
+      return ctx.resolveVersion(moduleName, getNames(dependencies), force)
+    },
+
+    /**
+     * Given the provided lists of build and service dependencies, return a list of all modules
+     * required to satisfy those dependencies.
+     */
+    async resolveModuleDependencies(buildDependencies: string[], serviceDependencies: string[]) {
+      const buildDeps = await Bluebird.map(buildDependencies, async (moduleName) => {
+        const module = await garden.getModule(moduleName)
+        const moduleDeps = await module.getBuildDependencies()
+        return [module].concat(await ctx.resolveModuleDependencies(getNames(moduleDeps), []))
+      })
+
+      const runtimeDeps = await Bluebird.map(serviceDependencies, async (serviceName) => {
+        const service = await garden.getService(serviceName)
+        const serviceDeps = await service.getDependencies()
+        return ctx.resolveModuleDependencies([service.module.name], getNames(serviceDeps))
+      })
+
+      const deps = flatten(buildDeps).concat(flatten(runtimeDeps))
+
+      return sortBy(uniqBy(deps, "name"), "name")
+    },
+
+    /**
+     * Given a module, and a list of dependencies, resolve the version for that combination of modules.
+     * The combined version is a either the latest dirty module version (if any), or the hash of the module version
+     * and the versions of its dependencies (in sorted order).
+     */
+    resolveVersion: async (moduleName: string, moduleDependencies: string[], force = false) => {
       const module = await ctx.getModule(moduleName)
       const cacheKey = ["moduleVersions", module.name]
 
       if (!force) {
-        const cached = <TreeVersion>garden.cache.get(cacheKey)
+        const cached = <ModuleVersion>garden.cache.get(cacheKey)
 
         if (cached) {
           return cached
         }
       }
 
-      const buildDependencies = await module.getBuildDependencies()
-      const cacheContexts = buildDependencies.concat([module]).map(m => m.getCacheContext())
+      const dependencies = await garden.getModules(moduleDependencies)
+      const cacheContexts = dependencies.concat([module]).map(m => m.getCacheContext())
 
+      // the version file is used internally to specify versions outside of source control
       const versionFilePath = join(module.path, GARDEN_VERSIONFILE_NAME)
       const versionFileContents = await pathExists(versionFilePath)
         && (await readFile(versionFilePath)).toString().trim()
 
-      let version: TreeVersion
+      let version: ModuleVersion
 
       if (!!versionFileContents) {
-        // this is used internally to specify version outside of source control
         try {
-          version = validate(JSON.parse(versionFileContents), versionFileSchema)
+          version = validate(JSON.parse(versionFileContents), moduleVersionSchema)
         } catch (err) {
           throw new ConfigurationError(
             `Unable to parse ${GARDEN_VERSIONFILE_NAME} as valid version file in module directory ${module.path}`,
@@ -468,27 +503,13 @@ export function createPluginContext(garden: Garden): PluginContext {
           )
         }
       } else {
-        const treeVersion = await garden.vcs.getTreeVersion([module.path])
-
-        const versionChain: TreeVersion[] = await Bluebird.map(
-          buildDependencies,
-          async (m: Module) => await m.getVersion(),
-        )
-        versionChain.push(treeVersion)
-
-        // The module version is the latest of any of the dependency modules or itself.
-        version = await ctx.getLatestVersion(versionChain)
+        version = await garden.vcs.resolveVersion(module, dependencies)
       }
 
       garden.cache.set(cacheKey, version, ...cacheContexts)
       return version
     },
 
-    getLatestVersion: async (versions: TreeVersion[]) => {
-      const sortedVersions = await garden.vcs.sortVersions(versions)
-      return sortedVersions[0]
-    },
-
     getStatus: async () => {
       const envStatus: EnvironmentStatusMap = await ctx.getEnvironmentStatus({})
       const services = keyBy(await ctx.getServices(), "name")

src/plugins/kubernetes/actions.ts

@@ -34,7 +34,7 @@ import {
   SetConfigParams,
   TestModuleParams,
 } from "../../types/plugin/params"
-import { TreeVersion } from "../../vcs/base"
+import { ModuleVersion } from "../../vcs/base"
 import {
   ContainerModule,
   helpers,
@@ -543,6 +543,6 @@ export async function logout({ ctx }: PluginActionParamsBase): Promise<LoginStat
   return { loggedIn: false }
 }
 
-function getTestResultKey(module: ContainerModule, testName: string, version: TreeVersion) {
+function getTestResultKey(module: ContainerModule, testName: string, version: ModuleVersion) {
   return `test-result--${module.name}--${testName}--${version.versionString}`
 }

src/plugins/kubernetes/specs-module.ts

@@ -42,7 +42,7 @@ import {
   TestConfig,
   TestSpec,
 } from "../../types/test"
-import { TreeVersion } from "../../vcs/base"
+import { ModuleVersion } from "../../vcs/base"
 import {
   applyMany,
 } from "./kubectl"
@@ -141,7 +141,7 @@ export const kubernetesSpecHandlers = {
   },
 }
 
-async function prepareSpecs(service: Service<KubernetesSpecsModule>, namespace: string, version: TreeVersion) {
+async function prepareSpecs(service: Service<KubernetesSpecsModule>, namespace: string, version: ModuleVersion) {
   return service.module.spec.specs.map((rawSpec) => {
     const spec = {
       metadata: {},

src/tasks/test.ts

@@ -8,11 +8,11 @@
 
 import * as Bluebird from "bluebird"
 import chalk from "chalk"
-import { uniqBy } from "lodash"
 import { PluginContext } from "../plugin-context"
 import { Module } from "../types/module"
 import { TestConfig } from "../types/test"
-import { TreeVersion } from "../vcs/base"
+import { getNames } from "../util/util"
+import { ModuleVersion } from "../vcs/base"
 import { BuildTask } from "./build"
 import { DeployTask } from "./deploy"
 import { TestResult } from "../types/plugin/outputs"
@@ -152,9 +152,8 @@ async function getTestDependencies(ctx: PluginContext, testConfig: TestConfig) {
 /**
  * Determine the version of the test run, based on the version of the module and each of its dependencies.
  */
-async function getTestVersion(ctx: PluginContext, module: Module, testConfig: TestConfig): Promise<TreeVersion> {
-  const dependencies = await getTestDependencies(ctx, testConfig)
-  const moduleDeps = uniqBy(dependencies.map(d => d.module).concat([module]), m => m.name)
-  const versions = await Bluebird.map(moduleDeps, m => m.getVersion())
-  return ctx.getLatestVersion(versions)
+async function getTestVersion(ctx: PluginContext, module: Module, testConfig: TestConfig): Promise<ModuleVersion> {
+  const buildDeps = await module.getBuildDependencies()
+  const moduleDeps = await ctx.resolveModuleDependencies(getNames(buildDeps), testConfig.dependencies)
+  return ctx.resolveVersion(module.name, getNames(moduleDeps))
 }

src/types/module.ts

@@ -10,7 +10,6 @@ import * as Bluebird from "bluebird"
 import * as Joi from "joi"
 import {
   flatten,
-  keyBy,
   set,
   uniq,
 } from "lodash"
@@ -23,7 +22,6 @@ import {
   TemplateStringContext,
 } from "../template-string"
 import { getNames } from "../util/util"
-import { TreeVersion } from "../vcs/base"
 import {
   joiArray,
   joiEnvVars,
@@ -85,14 +83,6 @@ export interface BuildConfig {
   dependencies: BuildDependencyConfig[],
 }
 
-export const versionFileSchema = Joi.object()
-  .keys({
-    versionString: Joi.string().required(),
-    latestCommit: Joi.string().required(),
-    dirtyTimestamp: Joi.number().allow(null).required(),
-  })
-  .meta({ internal: true })
-
 export interface ModuleSpec { }
 
 export interface BaseModuleSpec {
@@ -171,8 +161,6 @@ export class Module<
   public readonly services: ServiceConfig<S>[]
   public readonly tests: TestConfig<T>[]
 
-  private _buildDependencies: Module[]
-
   readonly _ConfigType: ModuleConfig<M>
 
   constructor(
@@ -218,7 +206,7 @@ export class Module<
     return pathToCacheContext(this.path)
   }
 
-  async getVersion(force?: boolean): Promise<TreeVersion> {
+  async getVersion(force?: boolean) {
     return this.ctx.getModuleVersion(this.name, force)
   }
 
@@ -227,32 +215,11 @@ export class Module<
   }
 
   async getBuildDependencies(): Promise<Module[]> {
-    if (this._buildDependencies) {
-      return this._buildDependencies
-    }
-
-    // TODO: Detect circular dependencies
-    const modules = keyBy(await this.ctx.getModules(), "name")
-    const deps: Module[] = []
-
-    for (let dep of this.config.build.dependencies) {
-      // TODO: find a more elegant way of dealing with plugin module dependencies
-      const dependencyName = dep.plugin ? `${dep.plugin}--${dep.name}` : dep.name
-      const dependency = modules[dependencyName]
-
-      if (!dependency) {
-        throw new ConfigurationError(`Module ${this.name} dependency ${dependencyName} not found`, {
-          module,
-          dependencyName,
-        })
-      }
-
-      deps.push(dependency)
-    }
-
-    this._buildDependencies = deps
-
-    return deps
+    // TODO: find a more elegant way of dealing with plugin module dependencies
+    const names = this.config.build.dependencies.map(
+      dep => dep.plugin ? `${dep.plugin}--${dep.name}` : dep.name,
+    )
+    return this.ctx.getModules(names)
   }
 
   async getServices(): Promise<Service[]> {

src/types/plugin/outputs.ts

@@ -7,7 +7,7 @@
  */
 
 import * as Joi from "joi"
-import { TreeVersion } from "../../vcs/base"
+import { ModuleVersion, moduleVersionSchema } from "../../vcs/base"
 import {
   joiArray,
   PrimitiveMap,
@@ -194,37 +194,21 @@ export const pushModuleResultSchema = Joi.object()
 export interface RunResult {
   moduleName: string
   command: string[]
-  version: TreeVersion
+  version: ModuleVersion
   success: boolean
   startedAt: Date
   completedAt: Date
   output: string
 }
 
-export const treeVersionSchema = Joi.object()
-  .keys({
-    versionString: Joi.string()
-      .required()
-      .description("String representation of the module version."),
-    latestCommit: Joi.string()
-      .required()
-      .description("The latest commit hash of the module source."),
-    dirtyTimestamp: Joi.number()
-      .allow(null)
-      .required()
-      .description(
-        "Set to the last modified time (as UNIX timestamp) if the module contains uncommitted changes, otherwise null.",
-    ),
-  })
-
 export const runResultSchema = Joi.object()
   .keys({
     moduleName: Joi.string()
       .description("The name of the module that was run."),
     command: Joi.array().items(Joi.string())
       .required()
       .description("The command that was run in the module."),
-    version: treeVersionSchema,
+    version: moduleVersionSchema,
     success: Joi.boolean()
       .required()
       .description("Whether the module was successfully run."),

src/types/plugin/params.ts

@@ -9,7 +9,7 @@
 import Stream from "ts-stream"
 import { LogEntry } from "../../logger/logger"
 import { PluginContext } from "../../plugin-context"
-import { TreeVersion } from "../../vcs/base"
+import { ModuleVersion } from "../../vcs/base"
 import {
   Environment,
   Primitive,
@@ -117,7 +117,7 @@ export interface TestModuleParams<T extends Module = Module> extends PluginModul
 
 export interface GetTestResultParams<T extends Module = Module> extends PluginModuleActionParamsBase<T> {
   testName: string
-  version: TreeVersion
+  version: ModuleVersion
 }
 
 export interface GetServiceStatusParams<T extends Module = Module> extends PluginServiceActionParamsBase<T> {