Merge pull request #241 from garden-io/multi-repos-hash-dirname

Multi-repo: ensure external source gets updated if repo url changes

garden-cli/package.json

@@ -32,6 +32,7 @@
     "chokidar": "^2.0.4",
     "cli-cursor": "^2.1.0",
     "cli-highlight": "^2.0.0",
+    "cli-truncate": "^1.1.0",
     "cryo": "0.0.6",
     "dedent": "^0.7.0",
     "deep-diff": "^1.0.1",
@@ -58,6 +59,7 @@
     "shx": "^0.3.2",
     "snyk": "^1.90.2",
     "split": "^1.0.1",
+    "string-width": "^2.1.1",
     "strip-ansi": "^4.0.0",
     "sywac": "^1.2.1",
     "terminal-link": "^1.1.0",
@@ -96,6 +98,7 @@
     "@types/node-emoji": "^1.8.0",
     "@types/path-is-inside": "^1.0.0",
     "@types/prettyjson": "0.0.28",
+    "@types/string-width": "^2.0.0",
     "@types/uniqid": "^4.1.2",
     "@types/wrap-ansi": "^3.0.0",
     "axios": "^0.18.0",

garden-cli/src/commands/update-remote/helpers.ts

@@ -7,26 +7,36 @@
  */
 
 import { difference } from "lodash"
-import { join } from "path"
+import { join, basename } from "path"
 import { remove, pathExists } from "fs-extra"
 
 import { getChildDirNames } from "../../util/util"
-import { ExternalSourceType, getRemoteSourcesDirName } from "../../util/ext-source-util"
+import {
+  ExternalSourceType,
+  getRemoteSourcesDirname,
+  getRemoteSourcePath,
+} from "../../util/ext-source-util"
+import { SourceConfig } from "../../config/project"
 
-export async function pruneRemoteSources({ projectRoot, names, type }: {
+export async function pruneRemoteSources({ projectRoot, sources, type }: {
   projectRoot: string,
-  names: string[],
+  sources: SourceConfig[],
   type: ExternalSourceType,
 }) {
-  const remoteSourcesPath = join(projectRoot, getRemoteSourcesDirName(type))
+  const remoteSourcesPath = join(projectRoot, getRemoteSourcesDirname(type))
 
   if (!(await pathExists(remoteSourcesPath))) {
     return
   }
 
-  const currentRemoteSourceNames = await getChildDirNames(remoteSourcesPath)
-  const staleRemoteSourceNames = difference(currentRemoteSourceNames, names)
-  for (const dirName of staleRemoteSourceNames) {
+  const sourceNames = sources
+    .map(({ name, repositoryUrl: url }) => getRemoteSourcePath({ name, url, sourceType: type }))
+    .map(srcPath => basename(srcPath))
+
+  const currentRemoteSources = await getChildDirNames(remoteSourcesPath)
+  const staleRemoteSources = difference(currentRemoteSources, sourceNames)
+
+  for (const dirName of staleRemoteSources) {
     await remove(join(remoteSourcesPath, dirName))
   }
 }

garden-cli/src/commands/update-remote/modules.ts

@@ -76,7 +76,7 @@ export class UpdateRemoteModulesCommand extends Command<typeof updateRemoteModul
       await ctx.vcs.updateRemoteSource({ name, url: repositoryUrl, sourceType: "module", logEntry: ctx.log })
     }
 
-    await pruneRemoteSources({ names, projectRoot: ctx.projectRoot, type: "module" })
+    await pruneRemoteSources({ projectRoot: ctx.projectRoot, type: "module", sources: moduleSources })
 
     return { result: moduleSources }
   }

garden-cli/src/commands/update-remote/sources.ts

@@ -72,7 +72,7 @@ export class UpdateRemoteSourcesCommand extends Command<typeof updateRemoteSourc
       await ctx.vcs.updateRemoteSource({ name, url: repositoryUrl, sourceType: "project", logEntry: ctx.log })
     }
 
-    await pruneRemoteSources({ names, projectRoot: ctx.projectRoot, type: "project" })
+    await pruneRemoteSources({ projectRoot: ctx.projectRoot, type: "project", sources: projectSources })
 
     return { result: projectSources }
   }

garden-cli/src/config/base.ts

@@ -12,7 +12,7 @@ import {
   getNames,
 } from "../util/util"
 import { baseModuleSpecSchema, ModuleConfig } from "./module"
-import { joiIdentifier, validate } from "./common"
+import { validate } from "./common"
 import { ConfigurationError } from "../exceptions"
 import * as Joi from "joi"
 import * as yaml from "js-yaml"
@@ -124,18 +124,6 @@ export async function loadConfig(projectRoot: string, path: string): Promise<Gar
       type: moduleConfig.type,
       variables: moduleConfig.variables,
     }
-
-    if (!moduleConfig.name) {
-      try {
-        moduleConfig.name = validate(dirname, joiIdentifier())
-      } catch (_) {
-        throw new ConfigurationError(
-          `Directory name ${parsed.dirname} is not a valid module name (must be valid identifier). ` +
-          `Please rename the directory or specify a module name in the garden.yml file.`,
-          { dirname: parsed.dirname },
-        )
-      }
-    }
   }
 
   return {

garden-cli/src/config/common.ts

@@ -82,6 +82,7 @@ export const joiRepositoryUrl = () => Joi
       "git",
       /git\+https?/,
       "https",
+      "file",
     ],
   })
   .description(

garden-cli/src/logger/renderers.ts

@@ -15,11 +15,13 @@ import {
   flow,
   isArray,
   isEmpty,
-  padEnd,
   padStart,
   reduce,
   kebabCase,
+  repeat,
 } from "lodash"
+import cliTruncate = require("cli-truncate")
+import stringWidth = require("string-width")
 import hasAnsi = require("has-ansi")
 
 import { LogSymbolType, EntryStyle } from "./types"
@@ -31,11 +33,13 @@ export type Renderers = Renderer[]
 
 /*** STYLE HELPERS ***/
 
-const sectionPrefixWidth = 18
-const truncate = (s: string) => s.length > sectionPrefixWidth
-  ? `${s.substring(0, sectionPrefixWidth - 3)}...`
-  : s
-const sectionStyle = (s: string) => chalk.cyan.italic(padEnd(truncate(s), sectionPrefixWidth))
+const SECTION_PREFIX_WIDTH = 25
+const cliPadEnd = (s: string, width: number): string => {
+  const diff = width - stringWidth(s)
+  return diff <= 0 ? s : s + repeat(" ", diff)
+}
+const truncateSection = (s: string) => cliTruncate(s, SECTION_PREFIX_WIDTH)
+const sectionStyle = (s: string) => chalk.cyan.italic(cliPadEnd(truncateSection(s), SECTION_PREFIX_WIDTH))
 const msgStyle = (s: string) => hasAnsi(s) ? s : chalk.gray(s)
 const errorStyle = chalk.red
 

garden-cli/src/tasks/deploy.ts

@@ -101,7 +101,7 @@ export class DeployTask extends Task {
       return status
     }
 
-    logEntry.setState({ section: this.service.name, msg: "Deploying" })
+    logEntry.setState("Deploying")
 
     const dependencies = await this.ctx.getServices(this.service.config.dependencies)
 

garden-cli/src/util/ext-source-util.ts

@@ -6,6 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
+import { createHash } from "crypto"
 import { uniqBy } from "lodash"
 import chalk from "chalk"
 import pathIsInside = require("path-is-inside")
@@ -21,16 +22,34 @@ import {
 import { ParameterError } from "../exceptions"
 import { Module } from "../types/module"
 import { PluginContext } from "../plugin-context"
+import { join } from "path"
 
 export type ExternalSourceType = "project" | "module"
 
-export function getRemoteSourcesDirName(type: ExternalSourceType): string {
+export function getRemoteSourcesDirname(type: ExternalSourceType): string {
   return type === "project" ? PROJECT_SOURCES_DIR_NAME : MODULE_SOURCES_DIR_NAME
 }
 
+/**
+ * A remote source dir name has the format 'source-name--HASH_OF_REPO_URL'
+ * so that we can detect if the repo url has changed
+ */
+export function getRemoteSourcePath({ name, url, sourceType }:
+  { name: string, url: string, sourceType: ExternalSourceType }) {
+  const dirname = name + "--" + hashRepoUrl(url)
+  return join(getRemoteSourcesDirname(sourceType), dirname)
+}
+
+export function hashRepoUrl(url: string) {
+  const urlHash = createHash("sha256")
+  urlHash.update(url)
+  return urlHash.digest("hex").slice(0, 10)
+}
+
 export function hasRemoteSource(module: Module): boolean {
   return !!module.repositoryUrl
 }
+
 export function getConfigKey(type: ExternalSourceType): string {
   return type === "project" ? localConfigKeys.linkedProjectSources : localConfigKeys.linkedModuleSources
 }

garden-cli/src/vcs/base.ts

@@ -15,7 +15,11 @@ import { join } from "path"
 import { GARDEN_VERSIONFILE_NAME } from "../constants"
 import { pathExists, readFile, writeFile } from "fs-extra"
 import { ConfigurationError } from "../exceptions"
-import { ExternalSourceType, getRemoteSourcesDirName } from "../util/ext-source-util"
+import {
+  ExternalSourceType,
+  getRemoteSourcesDirname,
+  getRemoteSourcePath,
+} from "../util/ext-source-util"
 import { LogNode } from "../logger/logger"
 import { ModuleConfig } from "../config/module"
 
@@ -159,10 +163,13 @@ export abstract class VcsHandler {
     }
   }
 
-  getRemoteSourcesDirName(type: ExternalSourceType) {
-    return getRemoteSourcesDirName(type)
+  getRemoteSourcesDirname(type: ExternalSourceType) {
+    return getRemoteSourcesDirname(type)
   }
 
+  getRemoteSourcePath(name, url, sourceType) {
+    return getRemoteSourcePath({ name, url, sourceType })
+  }
 }
 
 function hashVersions(versions: NamedTreeVersion[]) {

garden-cli/src/vcs/git.ts

@@ -6,30 +6,37 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-import { exec } from "child-process-promise"
+import execa = require("execa")
 import { join } from "path"
 import { ensureDir, pathExists, stat } from "fs-extra"
 import { argv } from "process"
 import Bluebird = require("bluebird")
-import { parse } from "url"
 
 import { NEW_MODULE_VERSION, VcsHandler, RemoteSourceParams } from "./base"
 import { EntryStyle } from "../logger/types"
 
 export const helpers = {
-  gitCli: (cwd: string): (args: string | string[]) => Promise<string> => {
-    return async args => {
-      const cmd = Array.isArray(args) ? `git ${args.join(" && git ")}` : `git ${args}`
-      const res = await exec(cmd, { cwd })
-      return res.stdout.trim()
+  gitCli: (cwd: string): (cmd: string, args: string[]) => Promise<string> => {
+    return async (cmd, args) => {
+      return execa.stdout("git", [cmd, ...args], { cwd })
     }
   },
 }
 
-function getUrlHash(url: string) {
-  return (parse(url).hash || "").split("#")[1]
+function getGitUrlParts(url: string) {
+  const parts = url.split("#")
+  return { repositoryUrl: parts[0], hash: parts[1] }
 }
 
+function parseRefList(res: string): string {
+  const refList = res.split("\n").map(str => {
+    const parts = str.split("\n")
+    return { commitId: parts[0], ref: parts[1] }
+  })
+  return refList[0].commitId
+}
+
+// TODO Consider moving git commands to separate (and testable) functions
 export class GitHandler extends VcsHandler {
   name = "git"
 
@@ -38,7 +45,12 @@ export class GitHandler extends VcsHandler {
 
     let commitHash
     try {
-      commitHash = await git("rev-list -1 --abbrev-commit --abbrev=10 HEAD") || NEW_MODULE_VERSION
+      commitHash = await git("rev-list", [
+        "--max-count=1",
+        "--abbrev-commit",
+        "--abbrev=10",
+        "HEAD",
+      ]) || NEW_MODULE_VERSION
     } catch (err) {
       if (err.code === 128) {
         // not in a repo root, return default version
@@ -48,13 +60,13 @@ export class GitHandler extends VcsHandler {
 
     let latestDirty = 0
 
-    const res = await git([`diff-index --name-only HEAD ${path}`, `ls-files --other --exclude-standard ${path}`])
+    const res = await git("diff-index", ["--name-only", "HEAD", path]) + "\n"
+      + await git("ls-files", ["--other", "--exclude-standard", path])
 
     const dirtyFiles: string[] = res.split("\n").filter((f) => f.length > 0)
     // for dirty trees, we append the last modified time of last modified or added file
     if (dirtyFiles.length) {
-
-      const repoRoot = await git("rev-parse --show-toplevel")
+      const repoRoot = await git("rev-parse", ["--show-toplevel"])
       const stats = await Bluebird.map(dirtyFiles, file => join(repoRoot, file))
         .filter((file: string) => pathExists(file))
         .map((file: string) => stat(file))
@@ -75,43 +87,56 @@ export class GitHandler extends VcsHandler {
 
   // TODO Better auth handling
   async ensureRemoteSource({ url, name, logEntry, sourceType }: RemoteSourceParams): Promise<string> {
-    const remoteSourcesPath = join(this.projectRoot, this.getRemoteSourcesDirName(sourceType))
+    const remoteSourcesPath = join(this.projectRoot, this.getRemoteSourcesDirname(sourceType))
     await ensureDir(remoteSourcesPath)
     const git = helpers.gitCli(remoteSourcesPath)
-    const fullPath = join(remoteSourcesPath, name)
 
-    if (!(await pathExists(fullPath))) {
+    const absPath = join(this.projectRoot, this.getRemoteSourcePath(name, url, sourceType))
+    const isCloned = await pathExists(absPath)
+
+    if (!isCloned) {
       const entry = logEntry.info({ section: name, msg: `Fetching from ${url}`, entryStyle: EntryStyle.activity })
-      const hash = getUrlHash(url)
-      const branch = hash ? `--branch=${hash}` : ""
+      const { repositoryUrl, hash } = getGitUrlParts(url)
 
-      await git(`clone --depth=1 ${branch} ${url} ${name}`)
+      const cmdOpts = ["--depth=1"]
+      if (hash) {
+        cmdOpts.push("--branch=hash")
+      }
+
+      await git("clone", [...cmdOpts, repositoryUrl, absPath])
 
       entry.setSuccess()
     }
 
-    return fullPath
+    return absPath
   }
 
   async updateRemoteSource({ url, name, sourceType, logEntry }: RemoteSourceParams) {
-    const sourcePath = join(this.projectRoot, this.getRemoteSourcesDirName(sourceType), name)
-    const git = helpers.gitCli(sourcePath)
+    const absPath = join(this.projectRoot, this.getRemoteSourcePath(name, url, sourceType))
+    const git = helpers.gitCli(absPath)
+    const { repositoryUrl, hash } = getGitUrlParts(url)
+
     await this.ensureRemoteSource({ url, name, sourceType, logEntry })
 
     const entry = logEntry.info({ section: name, msg: "Getting remote state", entryStyle: EntryStyle.activity })
-    await git("remote update")
+    await git("remote", ["update"])
+
+    const listRemoteArgs = hash ? [repositoryUrl, hash] : [repositoryUrl]
+    const showRefArgs = hash ? [hash] : []
+    const remoteCommitId = parseRefList(await git("ls-remote", listRemoteArgs))
+    const localCommitId = parseRefList(await git("show-ref", ["--hash", ...showRefArgs]))
 
-    const remoteHash = await git("rev-parse @")
-    const localHash = await git("rev-parse @{u}")
-    if (localHash !== remoteHash) {
+    if (localCommitId !== remoteCommitId) {
       entry.setState({ section: name, msg: `Fetching from ${url}`, entryStyle: EntryStyle.activity })
-      const hash = getUrlHash(url)
 
-      await git([`fetch origin ${hash} --depth=1`, `reset origin/${hash} --hard`])
+      const fetchArgs = hash ? ["origin", hash] : ["origin"]
+      const resetArgs = hash ? [`origin/${hash}`] : ["origin"]
+      await git("fetch", ["--depth=1", ...fetchArgs])
+      await git("reset", ["--hard", ...resetArgs])
 
       entry.setSuccess("Source updated")
     } else {
-      entry.setSuccess("Source up to date")
+      entry.setSuccess("Source already up to date")
     }
   }