Mutagen (dev mode) improvements

.circleci/config.yml

@@ -12,6 +12,7 @@ orbs:
   shared-env-config: &shared-env-config
     GARDEN_DISABLE_VERSION_CHECK: "true"
     GARDEN_DISABLE_ANALYTICS: "true"
+    GARDEN_K8S_BUILD_SYNC_MODE: "mutagen"
 
   # Configuration for our node jobs
   node-config: &node-config
@@ -604,7 +605,7 @@ jobs:
             # - Need to run with sudo to work with microk8s, because CircleCI doesn't allow us to log out
             #   and back in to add the circleci user to the microk8s group.
             # - We currently don't support in-cluster building on microk8s.
-            GARDEN_SKIP_TESTS="cluster-docker kaniko remote-only" sudo -E npm run integ
+            sudo -E GARDEN_SKIP_TESTS="cluster-docker kaniko remote-only" npm run _integ
       - run:
           name: Plugin tests
           command: sudo -E npm run test:plugins
@@ -673,7 +674,7 @@ jobs:
       - run:
           name: Integ tests
           # Note: We skip tests that only work for remote environments
-          command: cd core && yarn run integ-local
+          command: cd core && yarn integ-minikube
       - run:
           name: Plugin tests
           command: yarn run test:plugins

CONTRIBUTING.md

@@ -145,7 +145,7 @@ yarn test -- -g "taskGraph"  # run only tests with descriptions matching "taskGr
 Integration tests are run with:
 
 ```sh
-yarn run integ
+yarn integ-local
 ```
 
 End-to-end tests are run with:

core/package.json

@@ -247,17 +247,18 @@
     "typescript": "^4.3.5"
   },
   "scripts": {
+    "_integ": "mocha --opts test/mocha.integ.opts",
     "build": "gulp pegjs",
     "check-package-lock": "git diff-index --quiet HEAD -- yarn.lock || (echo 'yarn.lock is dirty!' && exit 1)",
     "clean": "shx rm -rf build",
     "dev": "gulp pegjs-watch",
     "fix-format": "prettier --write \"{src,test}/**/*.ts\"",
     "lint": "tslint -p .",
     "migration:generate": "typeorm migration:generate --config ormconfig.js -n",
-    "integ": "mocha --opts test/mocha.integ.opts",
-    "integ-kind": "GARDEN_INTEG_TEST_MODE=local GARDEN_SKIP_TESTS=\"cluster-docker cluster-buildkit cluster-buildkit-rootless kaniko remote-only\" mocha --opts test/mocha.integ.opts",
-    "integ-local": "GARDEN_INTEG_TEST_MODE=local GARDEN_SKIP_TESTS=remote-only mocha --opts test/mocha.integ.opts",
-    "integ-remote": "GARDEN_INTEG_TEST_MODE=remote GARDEN_SKIP_TESTS=local-only mocha --opts test/mocha.integ.opts",
+    "integ-kind": "GARDEN_INTEG_TEST_MODE=local GARDEN_SKIP_TESTS=\"cluster-docker cluster-buildkit cluster-buildkit-rootless kaniko remote-only\" yarn run _integ",
+    "integ-local": "GARDEN_LOGGER_TYPE=basic GARDEN_LOG_LEVEL=debug GARDEN_INTEG_TEST_MODE=local GARDEN_SKIP_TESTS=\"cluster-docker\" yarn run _integ",
+    "integ-minikube": "GARDEN_INTEG_TEST_MODE=local GARDEN_SKIP_TESTS=remote-only yarn run _integ",
+    "integ-remote": "GARDEN_INTEG_TEST_MODE=remote GARDEN_SKIP_TESTS=local-only yarn run _integ",
     "e2e": "cd test/e2e && ../../../bin/garden test",
     "e2e-project": "node build/test/e2e/e2e-project.js",
     "test": "mocha --opts test/mocha.opts"

core/src/constants.ts

@@ -60,6 +60,7 @@ export const gardenEnv = {
   GARDEN_ENABLE_PROFILING: env.get("GARDEN_ENABLE_PROFILING").required(false).asBool(),
   GARDEN_ENVIRONMENT: env.get("GARDEN_ENVIRONMENT").required(false).asString(),
   GARDEN_EXPERIMENTAL_BUILD_STAGE: env.get("GARDEN_EXPERIMENTAL_BUILD_STAGE").required(false).asBool(),
+  GARDEN_K8S_BUILD_SYNC_MODE: env.get("GARDEN_K8S_BUILD_SYNC_MODE").required(false).default("rsync").asString(),
   GARDEN_LEGACY_BUILD_STAGE: env.get("GARDEN_LEGACY_BUILD_STAGE").required(false).asBool(),
   GARDEN_LOG_LEVEL: env.get("GARDEN_LOG_LEVEL").required(false).asString(),
   GARDEN_LOGGER_TYPE: env.get("GARDEN_LOGGER_TYPE").required(false).asString(),

core/src/logger/renderers.ts

@@ -93,7 +93,7 @@ export function renderSymbolBasic(entry: LogEntry): string {
   let { symbol, status } = entry.getLatestMessage()
 
   if (symbol === "empty") {
-    return " "
+    return "  "
   }
   if (status === "active" && !symbol) {
     symbol = "info"
@@ -106,7 +106,7 @@ export function renderSymbol(entry: LogEntry): string {
   const { symbol } = entry.getLatestMessage()
 
   if (symbol === "empty") {
-    return " "
+    return "  "
   }
   return symbol ? `${logSymbols[symbol]} ` : ""
 }

core/src/plugin-context.ts

@@ -14,6 +14,8 @@ import { joi, joiVariables, joiStringMap, DeepPrimitiveMap } from "./config/comm
 import { PluginTool } from "./util/ext-tools"
 import { ConfigContext, ContextResolveOpts } from "./config/template-contexts/base"
 import { resolveTemplateStrings } from "./template-string/template-string"
+import { LogEntry } from "./logger/log-entry"
+import { logEntrySchema } from "./types/plugin/base"
 
 type WrappedFromGarden = Pick<
   Garden,
@@ -37,6 +39,7 @@ type ResolveTemplateStringsOpts = Omit<ContextResolveOpts, "stack">
 
 export interface PluginContext<C extends GenericProviderConfig = GenericProviderConfig> extends WrappedFromGarden {
   command: CommandInfo
+  log: LogEntry
   projectSources: SourceConfig[]
   provider: Provider<C>
   resolveTemplateStrings: <T>(o: T, opts?: ResolveTemplateStringsOpts) => T
@@ -64,6 +67,7 @@ export const pluginContextSchema = () =>
         The absolute path of the project's Garden dir. This is the directory the contains builds, logs and
         other meta data. A custom path can be set when initialising the Garden class. Defaults to \`.garden\`.
       `),
+      log: logEntrySchema(),
       production: joi
         .boolean()
         .default(false)
@@ -93,6 +97,7 @@ export async function createPluginContext(
     command,
     environmentName: garden.environmentName,
     gardenDirPath: garden.gardenDirPath,
+    log: garden.log,
     projectName: garden.projectName,
     projectRoot: garden.projectRoot,
     projectSources: garden.getProjectSources(),

core/src/plugins/container/config.ts

@@ -186,8 +186,18 @@ export interface DevModeSyncSpec {
   target: string
   mode: SyncMode
   exclude?: string[]
+  defaultFileMode?: number
+  defaultDirectoryMode?: number
+  defaultOwner?: number | string
+  defaultGroup?: number | string
 }
 
+const permissionsDocs =
+  "See the [Mutagen docs](https://mutagen.io/documentation/synchronization/permissions#permissions) for more information."
+
+const ownerDocs =
+  "Specify either an integer ID or a string name. See the [Mutagen docs](https://mutagen.io/documentation/synchronization/permissions#owners-and-groups) for more information."
+
 const devModeSyncSchema = () =>
   hotReloadSyncSchema().keys({
     exclude: joi
@@ -198,10 +208,33 @@ const devModeSyncSchema = () =>
     mode: joi
       .string()
       .allow("one-way", "one-way-replica", "two-way")
+      .only()
       .default("one-way")
       .description(
         "The sync mode to use for the given paths. Allowed options: `one-way`, `one-way-replica`, `two-way`."
       ),
+    defaultFileMode: joi
+      .number()
+      .min(0)
+      .max(0o777)
+      .description(
+        "The default permission bits, specified as an octal, to set on files at the sync target. Defaults to 0600 (user read/write). " +
+          permissionsDocs
+      ),
+    defaultDirectoryMode: joi
+      .number()
+      .min(0)
+      .max(0o777)
+      .description(
+        "The default permission bits, specified as an octal, to set on directories at the sync target. Defaults to 0700 (user read/write). " +
+          permissionsDocs
+      ),
+    defaultOwner: joi
+      .alternatives(joi.number().integer(), joi.string())
+      .description("Set the default owner of files and directories at the target. " + ownerDocs),
+    defaultGroup: joi
+      .alternatives(joi.number().integer(), joi.string())
+      .description("Set the default group on files and directories at the target. " + ownerDocs),
   })
 
 export interface ContainerDevModeSpec {
@@ -222,11 +255,11 @@ export const containerDevModeSchema = () =>
       .items(devModeSyncSchema())
       .description("Specify one or more source files or directories to automatically sync with the running container."),
   }).description(dedent`
-    **EXPERIMENTAL**
-
     Specifies which files or directories to sync to which paths inside the running containers of the service when it's in dev mode, and overrides for the container command and/or arguments.
 
     Dev mode is enabled when running the \`garden dev\` command, and by setting the \`--dev\` flag on the \`garden deploy\` command.
+
+    See the [Code Synchronization guide](https://docs.garden.io/guides/code-synchronization-dev-mode) for more information.
   `)
 
 export type ContainerServiceConfig = ServiceConfig<ContainerServiceSpec>

core/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts

@@ -309,9 +309,9 @@ async function runRegistryGarbageCollection(ctx: KubernetesPluginContext, api: K
       },
       {
         retries: 3,
-        onFailedAttempt: async () => {
-          log.warn("Failed to patch deployment, retrying in 5 seconds...")
-          await sleep(5)
+        minTimeout: 2000,
+        onFailedAttempt: async (err) => {
+          log.warn(`Failed to patch deployment. ${err.retriesLeft} attempts left.`)
         },
       }
     )

core/src/plugins/kubernetes/constants.ts

@@ -24,7 +24,7 @@ export const inClusterRegistryHostname = "127.0.0.1:5000"
 export const gardenUtilDaemonDeploymentName = "garden-util-daemon"
 export const dockerDaemonDeploymentName = "garden-docker-daemon"
 
-export const k8sUtilImageName = "gardendev/k8s-util:0.4.0"
+export const k8sUtilImageName = "gardendev/k8s-util:0.5.1"
 export const dockerDaemonContainerName = "docker-daemon"
 export const skopeoDaemonContainerName = "util"
 

core/src/plugins/kubernetes/container/build/buildkit.ts

@@ -96,6 +96,7 @@ export const buildkitBuildHandler: BuildHandler = async (params) => {
 
   const { contextPath } = await syncToBuildSync({
     ...params,
+    ctx: ctx as KubernetesPluginContext,
     api,
     namespace,
     deploymentName: buildkitDeploymentName,

core/src/plugins/kubernetes/container/build/cluster-docker.ts

@@ -85,6 +85,7 @@ export const clusterDockerBuild: BuildHandler = async (params) => {
 
   const { contextPath } = await syncToBuildSync({
     ...params,
+    ctx: ctx as KubernetesPluginContext,
     api,
     namespace: systemNamespace,
     deploymentName: sharedBuildSyncDeploymentName,

core/src/plugins/kubernetes/container/build/common.ts

@@ -21,7 +21,7 @@ import {
   rsyncPortName,
 } from "../../constants"
 import { KubeApi } from "../../api"
-import { KubernetesProvider } from "../../config"
+import { KubernetesPluginContext, KubernetesProvider } from "../../config"
 import { PodRunner } from "../../run"
 import { PluginContext } from "../../../../plugin-context"
 import { resolve } from "path"
@@ -34,10 +34,14 @@ import { getInClusterRegistryHostname } from "../../init"
 import { prepareDockerAuth } from "../../init"
 import chalk from "chalk"
 import { V1Container } from "@kubernetes/client-node"
+import { gardenEnv } from "../../../../constants"
+import { ensureMutagenSync, flushMutagenSync, getKubectlExecDestination, terminateMutagenSync } from "../../mutagen"
+import { randomString } from "../../../../util/string"
 
 const inClusterRegistryPort = 5000
 
 export const sharedBuildSyncDeploymentName = "garden-build-sync"
+export const utilContainerName = "util"
 export const utilRsyncPort = 8730
 
 export const commonSyncArgs = [
@@ -62,6 +66,7 @@ export type BuildStatusHandler = (params: GetBuildStatusParams<ContainerModule>)
 export type BuildHandler = (params: BuildModuleParams<ContainerModule>) => Promise<BuildResult>
 
 interface SyncToSharedBuildSyncParams extends BuildModuleParams<ContainerModule> {
+  ctx: KubernetesPluginContext
   api: KubeApi
   namespace: string
   deploymentName: string
@@ -71,39 +76,98 @@ interface SyncToSharedBuildSyncParams extends BuildModuleParams<ContainerModule>
 export async function syncToBuildSync(params: SyncToSharedBuildSyncParams) {
   const { ctx, module, log, api, namespace, deploymentName, rsyncPort } = params
 
+  // Because we're syncing to a shared volume, we need to scope by a unique ID
+  const contextPath = `/garden-build/${ctx.workingCopyId}/${module.name}/`
+
   const buildSyncPod = await getRunningDeploymentPod({
     api,
     deploymentName,
     namespace,
   })
-  // Sync the build context to the remote sync service
-  // -> Get a tunnel to the service
-  log.setState("Syncing sources to cluster...")
-  const syncFwd = await getPortForward({
-    ctx,
-    log,
-    namespace,
-    targetResource: `Pod/${buildSyncPod.metadata.name}`,
-    port: rsyncPort,
-  })
 
-  // -> Run rsync
-  const buildRoot = resolve(module.buildPath, "..")
-  // The '/./' trick is used to automatically create the correct target directory with rsync:
-  // https://stackoverflow.com/questions/1636889/rsync-how-can-i-configure-it-to-create-target-directory-on-server
-  let src = normalizeLocalRsyncPath(`${buildRoot}`) + `/./${module.name}/`
-  const destination = `rsync://localhost:${syncFwd.localPort}/volume/${ctx.workingCopyId}/`
-  const syncArgs = [...commonSyncArgs, "--relative", "--delete", "--temp-dir", "/tmp", src, destination]
-
-  log.debug(`Syncing from ${src} to ${destination}`)
-  // We retry a couple of times, because we may get intermittent connection issues or concurrency issues
-  await pRetry(() => exec("rsync", syncArgs), {
-    retries: 3,
-    minTimeout: 500,
-  })
+  if (gardenEnv.GARDEN_K8S_BUILD_SYNC_MODE === "mutagen") {
+    // Sync using mutagen
+    const key = `build-sync-${module.name}-${randomString(8)}`
+    const targetPath = `/data/${ctx.workingCopyId}/${module.name}`
+
+    // Make sure the target path exists
+    const runner = new PodRunner({
+      ctx,
+      provider: ctx.provider,
+      api,
+      pod: buildSyncPod,
+      namespace,
+    })
 
-  // Because we're syncing to a shared volume, we need to scope by a unique ID
-  const contextPath = `/garden-build/${ctx.workingCopyId}/${module.name}/`
+    await runner.exec({
+      log,
+      command: ["sh", "-c", "mkdir -p " + targetPath],
+      containerName: utilContainerName,
+      buffer: true,
+    })
+
+    try {
+      const resourceName = `Deployment/${deploymentName}`
+
+      log.debug(`Syncing from ${module.buildPath} to ${resourceName}`)
+
+      // -> Create the sync
+      await ensureMutagenSync({
+        log,
+        key,
+        logSection: module.name,
+        sourceDescription: `Module ${module.name} build path`,
+        targetDescription: "Build sync Pod",
+        config: {
+          alpha: module.buildPath,
+          beta: await getKubectlExecDestination({
+            ctx,
+            log,
+            namespace,
+            containerName: utilContainerName,
+            resourceName,
+            targetPath,
+          }),
+          mode: "one-way-replica",
+          ignore: [],
+        },
+      })
+
+      // -> Flush the sync once
+      await flushMutagenSync(log, key)
+      log.debug(`Sync from ${module.buildPath} to ${resourceName} completed`)
+    } finally {
+      // -> Terminate the sync
+      await terminateMutagenSync(log, key)
+      log.debug(`Sync connection terminated`)
+    }
+  } else {
+    // Sync the build context to the remote sync service
+    // -> Get a tunnel to the service
+    log.setState("Syncing sources to cluster...")
+    const syncFwd = await getPortForward({
+      ctx,
+      log,
+      namespace,
+      targetResource: `Pod/${buildSyncPod.metadata.name}`,
+      port: rsyncPort,
+    })
+
+    // -> Run rsync
+    const buildRoot = resolve(module.buildPath, "..")
+    // The '/./' trick is used to automatically create the correct target directory with rsync:
+    // https://stackoverflow.com/questions/1636889/rsync-how-can-i-configure-it-to-create-target-directory-on-server
+    let src = normalizeLocalRsyncPath(`${buildRoot}`) + `/./${module.name}/`
+    const destination = `rsync://localhost:${syncFwd.localPort}/volume/${ctx.workingCopyId}/`
+    const syncArgs = [...commonSyncArgs, "--relative", "--delete", "--temp-dir", "/tmp", src, destination]
+
+    log.debug(`Syncing from ${src} to ${destination}`)
+    // We retry a couple of times, because we may get intermittent connection issues or concurrency issues
+    await pRetry(() => exec("rsync", syncArgs), {
+      retries: 3,
+      minTimeout: 500,
+    })
+  }
 
   return { contextPath }
 }
@@ -288,7 +352,7 @@ function isLocalHostname(hostname: string) {
 
 export function getUtilContainer(authSecretName: string): V1Container {
   return {
-    name: "util",
+    name: utilContainerName,
     image: k8sUtilImageName,
     imagePullPolicy: "IfNotPresent",
     command: ["/rsync-server.sh"],

core/src/plugins/kubernetes/container/build/kaniko.ts

@@ -111,6 +111,7 @@ export const kanikoBuild: BuildHandler = async (params) => {
 
   await syncToBuildSync({
     ...params,
+    ctx: ctx as KubernetesPluginContext,
     api,
     namespace: projectNamespace,
     deploymentName: utilDeploymentName,