refactor: reduce work at module resolution time

Just a bit of debt being cleaned up here, but it does have a small
impact on init time.

The primary change was in the plugin interface, which is now a callback
that is resolved later in the init process.

cli/src/cli.ts

@@ -12,14 +12,15 @@ import { getDefaultProfiler } from "@garden-io/core/build/src/util/profiling"
 import { GardenProcess } from "@garden-io/core/build/src/db/entities/garden-process"
 import { ensureConnected } from "@garden-io/core/build/src/db/connection"
 import { GardenCli, RunOutput } from "@garden-io/core/build/src/cli/cli"
-import { GardenPlugin } from "@garden-io/core/build/src/types/plugin/plugin"
+import { GardenPluginCallback } from "@garden-io/core/build/src/types/plugin/plugin"
 
 // These plugins are always registered
-export const bundledPlugins = [
-  require("@garden-io/garden-conftest"),
-  require("@garden-io/garden-conftest-container"),
-  require("@garden-io/garden-conftest-kubernetes"),
-].map((m) => m.gardenPlugin as GardenPlugin)
+export const getBundledPlugins = (): GardenPluginCallback[] =>
+  [
+    require("@garden-io/garden-conftest"),
+    require("@garden-io/garden-conftest-container"),
+    require("@garden-io/garden-conftest-kubernetes"),
+  ].map((m) => () => m.gardenPlugin())
 
 export async function runCli({
   args,
@@ -38,7 +39,7 @@ export async function runCli({
 
   try {
     if (!cli) {
-      cli = new GardenCli({ plugins: bundledPlugins })
+      cli = new GardenCli({ plugins: getBundledPlugins() })
     }
     // Note: We slice off the binary/script name from argv.
     result = await cli.run({ args, exitOnError, processRecord })

cli/src/generate-docs.ts

@@ -11,7 +11,7 @@ import { resolve } from "path"
 import { Logger } from "@garden-io/core/build/src/logger/logger"
 import { LogLevel } from "@garden-io/core/build/src/logger/log-node"
 import { GARDEN_CLI_ROOT } from "@garden-io/core/build/src/constants"
-import { bundledPlugins } from "./cli"
+import { getBundledPlugins } from "./cli"
 
 require("source-map-support").install()
 
@@ -24,7 +24,7 @@ try {
   })
 } catch (_) {}
 
-generateDocs(resolve(GARDEN_CLI_ROOT, "..", "docs"), bundledPlugins)
+generateDocs(resolve(GARDEN_CLI_ROOT, "..", "docs"), getBundledPlugins())
   .then(() => {
     // tslint:disable-next-line: no-console
     console.log("Done!")

cli/test/unit/src/cli.ts

@@ -10,7 +10,7 @@ import { expect } from "chai"
 import { find } from "lodash"
 import { resolve } from "path"
 
-import { runCli, bundledPlugins } from "../../../src/cli"
+import { runCli, getBundledPlugins } from "../../../src/cli"
 import { testRoot } from "../../helpers"
 
 import { GardenCli } from "@garden-io/core/build/src/cli/cli"
@@ -29,7 +29,7 @@ describe("runCli", () => {
     const projectRoot = resolve(testRoot, "test-projects", "bundled-projects")
     const { cli, result } = await runCli({ args: ["tools", "--root", projectRoot], exitOnError: false })
 
-    expect(cli!["plugins"]).to.eql(bundledPlugins)
+    expect(cli!["plugins"].map((p) => p.name)).to.eql(getBundledPlugins().map((p) => p.name))
 
     const conftestTool = result?.result?.tools?.find((t) => t.pluginName === "conftest")
     expect(conftestTool).to.exist

core/src/cli/cli.ts

@@ -37,7 +37,7 @@ import { defaultDotIgnoreFiles } from "../util/fs"
 import { BufferedEventStream } from "../enterprise/buffered-event-stream"
 import { GardenProcess } from "../db/entities/garden-process"
 import { DashboardEventStream } from "../server/dashboard-event-stream"
-import { GardenPlugin } from "../types/plugin/plugin"
+import { GardenPluginCallback } from "../types/plugin/plugin"
 import { renderError } from "../logger/renderers"
 import { EnterpriseApi } from "../enterprise/api"
 
@@ -90,9 +90,9 @@ export interface RunOutput {
 export class GardenCli {
   private commands: { [key: string]: Command } = {}
   private fileWritersInitialized: boolean = false
-  private plugins: GardenPlugin[]
+  private plugins: GardenPluginCallback[]
 
-  constructor({ plugins }: { plugins?: GardenPlugin[] } = {}) {
+  constructor({ plugins }: { plugins?: GardenPluginCallback[] } = {}) {
     this.plugins = plugins || []
 
     const commands = sortBy(getAllCommands(), (c) => c.name)

core/src/commands/commands.ts

@@ -35,7 +35,7 @@ import { LogOutCommand } from "./logout"
 import { ToolsCommand } from "./tools"
 import { UtilCommand } from "./util/util"
 
-export const coreCommands: (Command | CommandGroup)[] = [
+export const getCoreCommands = (): (Command | CommandGroup)[] => [
   new BuildCommand(),
   new CallCommand(),
   new ConfigCommand(),
@@ -66,5 +66,5 @@ export const coreCommands: (Command | CommandGroup)[] = [
 ]
 
 export function getAllCommands() {
-  return coreCommands.flatMap((cmd) => (cmd instanceof CommandGroup ? [cmd, ...cmd.getSubCommands()] : [cmd]))
+  return getCoreCommands().flatMap((cmd) => (cmd instanceof CommandGroup ? [cmd, ...cmd.getSubCommands()] : [cmd]))
 }

core/src/commands/create/create-module.ts

@@ -18,7 +18,7 @@ import { resolve, basename, relative, join } from "path"
 import { GardenBaseError, ParameterError } from "../../exceptions"
 import { getModuleTypes, getPluginBaseNames } from "../../plugins"
 import { addConfig } from "./helpers"
-import { supportedPlugins } from "../../plugins/plugins"
+import { getSupportedPlugins } from "../../plugins/plugins"
 import { baseModuleSpecSchema } from "../../config/module"
 import { renderConfigReference } from "../../docs/config"
 import { DOCS_BASE_URL } from "../../constants"
@@ -119,7 +119,7 @@ export class CreateModuleCommand extends Command<CreateModuleArgs, CreateModuleO
       type,
     }
 
-    const allModuleTypes = getModuleTypes(supportedPlugins)
+    const allModuleTypes = getModuleTypes(getSupportedPlugins().map((p) => p()))
 
     if (opts.interactive && (!opts.name || !opts.type)) {
       log.root.stop()
@@ -219,7 +219,7 @@ export class CreateModuleCommand extends Command<CreateModuleArgs, CreateModuleO
         ])
 
         const allProvidersWithBases = flatten(
-          allProviders.map((p) => getPluginBaseNames(p.name, keyBy(supportedPlugins, "name")))
+          allProviders.map((p) => getPluginBaseNames(p.name, keyBy(getSupportedPlugins, "name")))
         )
 
         if (!allProvidersWithBases.includes(pluginName)) {

core/src/commands/tools.ts

@@ -88,7 +88,7 @@ export class ToolsCommand extends Command<Args, Opts> {
   }
 
   async action({ garden, log, args, opts }: CommandParams<Args>) {
-    const tools = getTools(garden)
+    const tools = await getTools(garden)
 
     if (!args.tool) {
       // We're listing tools, not executing one
@@ -113,7 +113,7 @@ export class ToolsCommand extends Command<Args, Opts> {
     }
 
     // We're executing a tool
-    const availablePlugins = Object.values(garden["registeredPlugins"])
+    const availablePlugins = await garden.getAllPlugins()
     let plugins = availablePlugins
 
     if (pluginName) {
@@ -136,7 +136,7 @@ export class ToolsCommand extends Command<Args, Opts> {
             log.debug(`Unable to resolve project config: ${err.message}`)
           }
         }
-        const configuredPlugins = await garden.getPlugins()
+        const configuredPlugins = await garden.getAllPlugins()
         plugins = uniqByName([...configuredPlugins, ...availablePlugins])
       }
     }
@@ -180,15 +180,15 @@ export class ToolsCommand extends Command<Args, Opts> {
   }
 }
 
-function getTools(garden: Garden) {
-  const registeredPlugins = Object.values(garden["registeredPlugins"])
+async function getTools(garden: Garden) {
+  const registeredPlugins = await garden.getAllPlugins()
 
   return sortBy(registeredPlugins, "name").flatMap((plugin) =>
     (plugin.tools || []).map((tool) => ({ ...omit(tool, "_includeInGardenImage"), pluginName: plugin.name }))
   )
 }
 
-function printTools(garden: Garden, log: LogEntry) {
+async function printTools(garden: Garden, log: LogEntry) {
   log.info(dedent`
   ${chalk.white.bold("USAGE")}
 
@@ -198,7 +198,7 @@ function printTools(garden: Garden, log: LogEntry) {
   ${chalk.white.bold("PLUGIN TOOLS")}
   `)
 
-  const tools = getTools(garden)
+  const tools = await getTools(garden)
 
   const rows = tools.map((tool) => {
     return [

core/src/commands/util/fetch-tools.ts

@@ -58,7 +58,7 @@ export class FetchToolsCommand extends Command<{}, FetchToolsOpts> {
     let plugins: GardenPlugin[]
 
     if (opts.all) {
-      plugins = Object.values(garden.registeredPlugins)
+      plugins = await garden.getAllPlugins()
       printHeader(log, "Fetching tools for all registered providers", "hammer_and_wrench")
     } else {
       const projectRoot = findProjectConfig(garden.projectRoot)
@@ -74,7 +74,7 @@ export class FetchToolsCommand extends Command<{}, FetchToolsOpts> {
         garden = await Garden.factory(garden.projectRoot, { ...omit(garden.opts, "config"), log })
       }
 
-      plugins = await garden.getPlugins()
+      plugins = await garden.getConfiguredPlugins()
 
       printHeader(log, "Fetching all tools for the current project and environment", "hammer_and_wrench")
     }

core/src/config/base.ts

@@ -172,7 +172,7 @@ export function prepareModuleResource(spec: any, configPath: string, projectRoot
     repositoryUrl: spec.repositoryUrl,
     serviceConfigs: [],
     spec: {
-      ...omit(spec, baseModuleSchemaKeys),
+      ...omit(spec, baseModuleSchemaKeys()),
       build: { ...spec.build, dependencies },
     },
     testConfigs: [],

core/src/config/module.ts

@@ -267,17 +267,18 @@ export const moduleConfigSchema = () =>
     .description("The configuration for a module.")
     .unknown(false)
 
-export const baseModuleSchemaKeys = Object.keys(baseModuleSpecSchema().describe().keys).concat([
-  "kind",
-  "name",
-  "type",
-  "path",
-  "configPath",
-  "serviceConfigs",
-  "taskConfigs",
-  "testConfigs",
-  "_config",
-])
+export const baseModuleSchemaKeys = () =>
+  Object.keys(baseModuleSpecSchema().describe().keys).concat([
+    "kind",
+    "name",
+    "type",
+    "path",
+    "configPath",
+    "serviceConfigs",
+    "taskConfigs",
+    "testConfigs",
+    "_config",
+  ])
 
 export function serializeConfig(moduleConfig: Partial<ModuleConfig>) {
   return stableStringify(moduleConfig)

core/src/config/workflow.ts

@@ -16,7 +16,7 @@ import { WorkflowConfigContext } from "./config-context"
 import { resolveTemplateStrings } from "../template-string"
 import { validateWithPath } from "./validation"
 import { ConfigurationError } from "../exceptions"
-import { coreCommands } from "../commands/commands"
+import { getCoreCommands } from "../commands/commands"
 import { CommandGroup } from "../commands/base"
 import { EnvironmentConfig, getNamespace } from "./project"
 import { globalOptions } from "../cli/params"
@@ -335,7 +335,7 @@ export function resolveWorkflowConfig(garden: Garden, config: WorkflowConfig) {
  * Get all commands whitelisted for workflows
  */
 function getStepCommands() {
-  return coreCommands
+  return getCoreCommands()
     .flatMap((cmd) => {
       if (cmd instanceof CommandGroup) {
         return cmd.getSubCommands()

core/src/docs/commands.ts

@@ -10,15 +10,15 @@ import { readFileSync, writeFileSync } from "fs"
 import handlebars from "handlebars"
 import { resolve } from "path"
 import { globalOptions } from "../cli/params"
-import { coreCommands } from "../commands/commands"
+import { getCoreCommands } from "../commands/commands"
 import { describeParameters, CommandGroup } from "../commands/base"
 import { TEMPLATES_DIR, renderConfigReference } from "./config"
 
 export function writeCommandReferenceDocs(docsRoot: string) {
   const referenceDir = resolve(docsRoot, "reference")
   const outputPath = resolve(referenceDir, "commands.md")
 
-  const commands = coreCommands
+  const commands = getCoreCommands()
     .flatMap((cmd) => {
       if (cmd instanceof CommandGroup && cmd.subCommands?.length) {
         return cmd

core/src/docs/generate.ts

@@ -19,18 +19,21 @@ import { writeFileSync, readFile, writeFile } from "fs-extra"
 import { renderModuleTypeReference, moduleTypes } from "./module-type"
 import { renderProviderReference } from "./provider"
 import { defaultNamespace } from "../config/project"
-import { GardenPlugin } from "../types/plugin/plugin"
+import { GardenPlugin, GardenPluginCallback } from "../types/plugin/plugin"
 import { workflowConfigSchema } from "../config/workflow"
 import { moduleTemplateSchema } from "../config/module-template"
 
-export async function generateDocs(targetDir: string, plugins: GardenPlugin[]) {
+export async function generateDocs(targetDir: string, plugins: GardenPluginCallback[]) {
   // tslint:disable: no-console
   const docsRoot = resolve(process.cwd(), targetDir)
 
   console.log("Updating command references...")
   writeCommandReferenceDocs(docsRoot)
   console.log("Updating config references...")
-  await writeConfigReferenceDocs(docsRoot, plugins)
+  await writeConfigReferenceDocs(
+    docsRoot,
+    plugins.map((p) => p())
+  )
   console.log("Updating template string reference...")
   writeTemplateStringReferenceDocs(docsRoot)
   console.log("Generating table of contents...")
@@ -77,7 +80,7 @@ export async function writeConfigReferenceDocs(docsRoot: string, plugins: Garden
   })
 
   const providerDir = resolve(docsRoot, "reference", "providers")
-  const allPlugins = await garden.getPlugins()
+  const allPlugins = await garden.getAllPlugins()
   const pluginsByName = keyBy(allPlugins, "name")
   const providersReadme = ["---", "order: 1", "title: Providers", "---", "", "# Providers", ""]