feat(vcs): add new git repo scanning method to improve resolution speed

.gardenignore

@@ -1,3 +1,7 @@
+.garden/
+.history/
+.vscode/
+.gradle/
 node_modules/
 tmp/
 core/tmp/

.testignore

@@ -0,0 +1,6 @@
+.garden/
+.history/
+.vscode/
+.gradle/
+tmp/
+core/tmp/

cli/test/unit/src/cli.ts

@@ -13,8 +13,8 @@ import { resolve } from "path"
 import { runCli, getBundledPlugins } from "../../../src/cli"
 import { testRoot } from "../../helpers"
 
-import { GardenCli } from "@garden-io/core/build/src/cli/cli"
 import { projectRootA } from "@garden-io/core/build/test/helpers"
+import { TestGardenCli } from "@garden-io/core/build/test/helpers/cli"
 import { Command, CommandParams } from "@garden-io/core/build/src/commands/base"
 import { randomString } from "@garden-io/core/build/src/util/string"
 import { GlobalConfigStore } from "@garden-io/core/build/src/config-store/global"
@@ -59,7 +59,7 @@ describe("runCli", () => {
       }
     }
 
-    const cli = new GardenCli()
+    const cli = new TestGardenCli()
     const cmd = new TestCommand()
     cli.addCommand(cmd)
 
@@ -84,7 +84,7 @@ describe("runCli", () => {
       }
     }
 
-    const cli = new GardenCli()
+    const cli = new TestGardenCli()
     const cmd = new TestCommand()
     cli.addCommand(cmd)
 

core/package.json

@@ -263,7 +263,8 @@
     "integ-local": "GARDEN_INTEG_TEST_MODE=local GARDEN_SKIP_TESTS=\"remote-only\" yarn _integ",
     "integ-minikube": "GARDEN_INTEG_TEST_MODE=local GARDEN_SKIP_TESTS=\"remote-only\" yarn _integ",
     "integ-remote": "GARDEN_INTEG_TEST_MODE=remote GARDEN_SKIP_TESTS=local-only yarn _integ",
-    "test": "mocha"
+    "test": "mocha",
+    "test:silly": "GARDEN_LOGGER_TYPE=basic GARDEN_LOG_LEVEL=silly mocha"
   },
   "pkg": {
     "scripts": [

core/src/cli/cli.ts

@@ -37,7 +37,7 @@ import { ERROR_LOG_FILENAME, DEFAULT_GARDEN_DIR_NAME, LOGS_DIR_NAME, gardenEnv }
 import { generateBasicDebugInfoReport } from "../commands/get/get-debug-info"
 import { AnalyticsHandler } from "../analytics/analytics"
 import { GardenPluginReference } from "../plugin/plugin"
-import { CloudApi, CloudApiTokenRefreshError, getGardenCloudDomain } from "../cloud/api"
+import { CloudApi, CloudApiFactory, CloudApiTokenRefreshError, getGardenCloudDomain } from "../cloud/api"
 import { findProjectConfig } from "../config/base"
 import { pMemoizeDecorator } from "../lib/p-memoize"
 import { getCustomCommands } from "../commands/custom"
@@ -61,6 +61,12 @@ export interface RunOutput {
   consoleOutput?: string
 }
 
+export interface GardenCliParams {
+  plugins?: GardenPluginReference[]
+  initLogger?: boolean
+  cloudApiFactory?: CloudApiFactory
+}
+
 // TODO: this is used in more contexts now, should rename to GardenCommandRunner or something like that
 @Profile()
 export class GardenCli {
@@ -69,10 +75,12 @@ export class GardenCli {
   public plugins: GardenPluginReference[]
   private initLogger: boolean
   public processRecord: GardenProcess
+  protected cloudApiFactory: CloudApiFactory
 
-  constructor({ plugins, initLogger = false }: { plugins?: GardenPluginReference[]; initLogger?: boolean } = {}) {
+  constructor({ plugins, initLogger = false, cloudApiFactory = CloudApi.factory }: GardenCliParams = {}) {
     this.plugins = plugins || []
     this.initLogger = initLogger
+    this.cloudApiFactory = cloudApiFactory
 
     const commands = sortBy(getBuiltinCommands(), (c) => c.name)
     commands.forEach((command) => this.addCommand(command))
@@ -223,7 +231,7 @@ ${renderCommands(commands)}
         const distroName = getCloudDistributionName(cloudDomain)
 
         try {
-          cloudApi = await CloudApi.factory({ log, cloudDomain, globalConfigStore })
+          cloudApi = await this.cloudApiFactory({ log, cloudDomain, globalConfigStore })
         } catch (err) {
           if (err instanceof CloudApiTokenRefreshError) {
             log.warn(dedent`

core/src/cloud/api.ts

@@ -20,13 +20,15 @@ import {
   GetProfileResponse,
   CreateProjectsForRepoResponse,
   ListProjectsResponse,
+  BaseResponse,
 } from "@garden-io/platform-api-types"
 import { getCloudDistributionName, getCloudLogSectionName, getPackageVersion } from "../util/util"
 import { CommandInfo } from "../plugin-context"
 import type { ClientAuthToken, GlobalConfigStore } from "../config-store/global"
 import { add } from "date-fns"
 import { LogLevel } from "../logger/logger"
 import { makeAuthHeader } from "./auth"
+import { StringMap } from "../config/common"
 
 const gardenClientName = "garden-core"
 const gardenClientVersion = getPackageVersion()
@@ -117,6 +119,12 @@ export interface CloudProject {
   environments: CloudEnvironment[]
 }
 
+export interface GetSecretsParams {
+  log: Log
+  projectId: string
+  environmentName: string
+}
+
 function toCloudProject(
   project: GetProjectResponse["data"] | ListProjectsResponse["data"][0] | CreateProjectsForRepoResponse["data"][0]
 ): CloudProject {
@@ -157,6 +165,8 @@ export interface CloudApiFactoryParams {
   skipLogging?: boolean
 }
 
+export type CloudApiFactory = (params: CloudApiFactoryParams) => Promise<CloudApi | undefined>
+
 /**
  * The Enterprise API client.
  *
@@ -724,4 +734,40 @@ export class CloudApi {
   getRegisteredSession(sessionId: string) {
     return this.registeredSessions.get(sessionId)
   }
+
+  async getSecrets({ log, projectId, environmentName }: GetSecretsParams): Promise<StringMap> {
+    let secrets: StringMap = {}
+    const distroName = getCloudDistributionName(this.domain)
+
+    try {
+      const res = await this.get<BaseResponse>(`/secrets/projectUid/${projectId}/env/${environmentName}`)
+      secrets = res.data
+    } catch (err) {
+      if (isGotError(err, 404)) {
+        log.debug(`No secrets were received from ${distroName}.`)
+        log.debug("")
+        log.debug(deline`
+          Either the environment ${environmentName} does not exist in ${distroName}, or no project
+          with the id in your project configuration exists in ${distroName}.
+        `)
+        log.debug("")
+        log.debug(deline`
+          Please visit ${this.domain} to review the environments and projects currently
+          in the system.
+        `)
+      } else {
+        throw err
+      }
+    }
+
+    const emptyKeys = Object.keys(secrets).filter((key) => !secrets[key])
+    if (emptyKeys.length > 0) {
+      const prefix =
+        emptyKeys.length === 1
+          ? "The following secret key has an empty value"
+          : "The following secret keys have empty values"
+      log.error(`${prefix}: ${emptyKeys.sort().join(", ")}`)
+    }
+    return secrets
+  }
 }

core/src/commands/sync/sync-start.ts

@@ -241,8 +241,8 @@ export async function startSyncWithoutDeploy({
         someSyncStarted = true
 
         if (monitor) {
-          const monitor = new SyncMonitor({ garden, log, action: executedAction, graph, stopOnExit })
-          garden.monitors.addAndSubscribe(monitor, command)
+          const m = new SyncMonitor({ garden, log, action: executedAction, graph, stopOnExit })
+          garden.monitors.addAndSubscribe(m, command)
         }
       } catch (error) {
         actionLog.warn(

core/src/config/project.ts

@@ -30,7 +30,7 @@ import { findByName, getNames } from "../util/util"
 import { ConfigurationError, ParameterError, ValidationError } from "../exceptions"
 import { cloneDeep, memoize } from "lodash"
 import { GenericProviderConfig, providerConfigBaseSchema } from "./provider"
-import { DOCS_BASE_URL, GardenApiVersion } from "../constants"
+import { DOCS_BASE_URL, GardenApiVersion, GitScanMode, gitScanModes } from "../constants"
 import { defaultDotIgnoreFile } from "../util/fs"
 import type { CommandInfo } from "../plugin-context"
 import type { VcsInfo } from "../vcs/vcs"
@@ -191,6 +191,10 @@ export interface ProxyConfig {
   hostname: string
 }
 
+interface GitConfig {
+  mode: GitScanMode
+}
+
 export interface ProjectConfig {
   apiVersion: GardenApiVersion
   kind: "Project"
@@ -207,6 +211,7 @@ export interface ProjectConfig {
   scan?: {
     include?: string[]
     exclude?: string[]
+    git?: GitConfig
   }
   outputs?: OutputSpec[]
   providers: GenericProviderConfig[]
@@ -261,6 +266,16 @@ const projectScanSchema = createSchema({
       `
       )
       .example(["public/**/*", "tmp/**/*"]),
+    git: joi.object().keys({
+      mode: joi
+        .string()
+        .allow(...gitScanModes)
+        .only()
+        .default("subtree")
+        .description(
+          "Choose how to perform scans of git repositories. The default (`subtree`) runs individual git scans on each action/module path. The `repo` mode scans entire repositories and then filters down to files matching the paths, includes and excludes for each action/module. This can be considerably more efficient for large projects with many actions/modules."
+        ),
+    }),
   }),
 })
 
@@ -376,7 +391,7 @@ export const projectSchema = createSchema({
         )
         .example(["127.0.0.1"]),
     }),
-    scan: projectScanSchema().description("Control where to scan for configuration files in the project."),
+    scan: projectScanSchema().description("Control where and how to scan for configuration files in the project."),
     outputs: joiSparseArray(projectOutputSchema())
       .unique("name")
       .description(

core/src/constants.ts

@@ -12,6 +12,10 @@ import { homedir } from "os"
 
 export const isPkg = !!(<any>process).pkg
 
+export const gitScanModes = ["repo", "subtree"] as const
+export type GitScanMode = (typeof gitScanModes)[number]
+export const defaultGitScanMode: GitScanMode = "subtree"
+
 export const GARDEN_CORE_ROOT = isPkg ? resolve(process.execPath, "..") : resolve(__dirname, "..", "..")
 export const GARDEN_CLI_ROOT = isPkg ? resolve(process.execPath, "..") : resolve(GARDEN_CORE_ROOT, "..", "cli")
 export const STATIC_DIR = isPkg ? resolve(process.execPath, "..", "static") : resolve(GARDEN_CORE_ROOT, "..", "static")
@@ -62,6 +66,7 @@ export const gardenEnv = {
   GARDEN_ENVIRONMENT: env.get("GARDEN_ENVIRONMENT").required(false).asString(),
   GARDEN_EXPERIMENTAL_BUILD_STAGE: env.get("GARDEN_EXPERIMENTAL_BUILD_STAGE").required(false).asBool(),
   GARDEN_GE_SCHEDULED: env.get("GARDEN_GE_SCHEDULED").required(false).asBool(),
+  GARDEN_GIT_SCAN_MODE: env.get("GARDEN_GIT_SCAN_MODE").required(false).asEnum(gitScanModes),
   GARDEN_LEGACY_BUILD_STAGE: env.get("GARDEN_LEGACY_BUILD_STAGE").required(false).asBool(),
   GARDEN_LOG_LEVEL: env.get("GARDEN_LOG_LEVEL").required(false).asString(),
   GARDEN_LOGGER_TYPE: env.get("GARDEN_LOGGER_TYPE").required(false).asString(),