perf(core): cache provider statuses for faster successive startup

We now cache provider statuses for a default of one hour. You can run
any command with `--force-refresh` to skip the caching, and can override
the cache duration with the `GARDEN_CACHE_TTL=<seconds>` environment
variable.

This substantially improves command execution times when run in
succession, which will be very useful for day-to-day usage, as well CI
performance.

Closes #1824

docs/reference/commands.md

@@ -30,6 +30,7 @@ quiet: suppresses all log output, same as --silent.
   | `--output` | `-o` | `json` `yaml`  | Output command result in specified format (note: disables progress logging and interactive functionality).
   | `--emoji` |  | boolean | Enable emoji in output (defaults to true if the environment supports it).
   | `--yes` | `-y` | boolean | Automatically approve any yes/no prompts during execution.
+  | `--force-refresh` |  | boolean | Force refresh of any caches, e.g. cached provider statuses.
 
 ### garden build
 

garden-service/src/cli/cli.ts

@@ -164,6 +164,10 @@ export const GLOBAL_OPTIONS = {
     help: "Automatically approve any yes/no prompts during execution.",
     defaultValue: false,
   }),
+  "force-refresh": new BooleanParameter({
+    help: "Force refresh of any caches, e.g. cached provider statuses.",
+    defaultValue: false,
+  }),
 }
 
 export type GlobalOptions = typeof GLOBAL_OPTIONS
@@ -290,6 +294,7 @@ export class GardenCli {
         "env": environmentName,
         silent,
         output,
+        "force-refresh": forceRefresh,
       } = parsedOpts
 
       let loggerType = loggerTypeOpt || command.getLoggerType({ opts: parsedOpts, args: parsedArgs })
@@ -323,6 +328,7 @@ export class GardenCli {
         environmentName,
         log,
         sessionId,
+        forceRefresh,
       }
 
       let garden: Garden

garden-service/src/config/status.ts

@@ -61,5 +61,14 @@ export const environmentStatusSchema = () =>
       outputs: joiVariables()
         .meta({ extendable: true })
         .description("Output variables that modules and other variables can reference."),
+      disableCache: joi
+        .boolean()
+        .optional()
+        .description("Set to true to disable caching of the status."),
+      cached: joi
+        .boolean()
+        .optional()
+        .meta({ internal: true })
+        .description("Indicates if the status was retrieved from cache by the framework."),
     })
     .description("Description of an environment's status for a provider.")

garden-service/src/garden.ts

@@ -95,6 +95,7 @@ export interface GardenOpts {
   commandInfo?: CommandInfo
   gardenDirPath?: string
   environmentName?: string
+  forceRefresh?: boolean
   persistent?: boolean
   log?: LogEntry
   plugins?: RegisterPluginParam[]
@@ -130,6 +131,7 @@ export interface GardenParams {
   username: string | undefined
   vcs: VcsHandler
   workingCopyId: string
+  forceRefresh?: boolean
 }
 
 @Profile()
@@ -182,6 +184,7 @@ export class Garden {
   public readonly systemNamespace: string
   public readonly username?: string
   public readonly version: ModuleVersion
+  private readonly forceRefresh: boolean
 
   constructor(params: GardenParams) {
     this.buildDir = params.buildDir
@@ -212,6 +215,7 @@ export class Garden {
     this.persistent = !!params.opts.persistent
     this.username = params.username
     this.vcs = params.vcs
+    this.forceRefresh = !!params.forceRefresh
 
     // make sure we're on a supported platform
     const currentPlatform = platform()
@@ -386,6 +390,7 @@ export class Garden {
       log,
       username: _username,
       vcs,
+      forceRefresh: opts.forceRefresh,
     }) as InstanceType<T>
 
     return garden
@@ -613,6 +618,7 @@ export class Garden {
           plugin,
           config,
           version: this.version,
+          forceRefresh: this.forceRefresh,
           forceInit,
         })
       })
@@ -634,6 +640,8 @@ export class Garden {
 
       providers = Object.values(taskResults).map((result) => result!.output)
 
+      const gotCachedResult = !!providers.find((p) => p.status.cached)
+
       await Bluebird.map(providers, async (provider) =>
         Bluebird.map(provider.moduleConfigs, async (moduleConfig) => {
           // Make sure module and all nested entities are scoped to the plugin
@@ -646,7 +654,16 @@ export class Garden {
         this.resolvedProviders[provider.name] = provider
       }
 
-      log.setSuccess({ msg: chalk.green("Done"), append: true })
+      if (gotCachedResult) {
+        log.setSuccess({ msg: chalk.green("Cached"), append: true })
+        log.info({
+          symbol: "info",
+          msg: chalk.gray("Run with --force-refresh to force a refresh of provider statuses."),
+        })
+      } else {
+        log.setSuccess({ msg: chalk.green("Done"), append: true })
+      }
+
       this.log.silly(`Resolved providers: ${providers.map((p) => p.name).join(", ")}`)
     })
 

garden-service/src/plugins/terraform/commands.ts

@@ -14,8 +14,10 @@ import { prepareVariables, tfValidate } from "./common"
 import { Module } from "../../types/module"
 import { findByName } from "../../util/util"
 import { TerraformModule } from "./module"
-import { PluginCommand } from "../../types/plugin/command"
+import { PluginCommand, PluginCommandParams } from "../../types/plugin/command"
 import { join } from "path"
+import { remove } from "fs-extra"
+import { getProviderStatusCachePath } from "../../tasks/resolve-provider"
 
 const commandsToWrap = ["apply", "plan"]
 const initCommand = chalk.bold("terraform init")
@@ -32,7 +34,7 @@ function makeRootCommand(commandName: string) {
     name: commandName + "-root",
     description: `Runs ${terraformCommand} for the provider root stack, with the provider variables automatically configured as inputs. Positional arguments are passed to the command. If necessary, ${initCommand} is run first.`,
     title: chalk.bold.magenta(`Running ${chalk.white.bold(terraformCommand)} for project root stack`),
-    async handler({ ctx, args, log }) {
+    async handler({ ctx, args, log }: PluginCommandParams) {
       const provider = ctx.provider as TerraformProvider
 
       if (!provider.config.initRoot) {
@@ -41,6 +43,14 @@ function makeRootCommand(commandName: string) {
         })
       }
 
+      // Clear the provider status cache, to avoid any user confusion
+      const cachePath = getProviderStatusCachePath({
+        gardenDirPath: ctx.gardenDirPath,
+        pluginName: provider.name,
+        environmentName: ctx.environmentName,
+      })
+      await remove(cachePath)
+
       const root = join(ctx.projectRoot, provider.config.initRoot)
 
       await tfValidate(log, provider, root, provider.config.variables)

garden-service/src/plugins/terraform/init.ts

@@ -43,7 +43,8 @@ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusPar
         `),
       })
       const outputs = await getTfOutputs(log, tfVersion, root)
-      return { ready: true, outputs }
+      // Make sure the status is not cached when the stack is not up-to-date
+      return { ready: true, outputs, disableCache: true }
     }
   } else {
     return { ready: false, outputs: {} }

garden-service/src/tasks/resolve-provider.ts

@@ -6,6 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
+import stableStringify = require("json-stable-stringify")
 import chalk from "chalk"
 import { BaseTask, TaskParams, TaskType } from "./base"
 import { ProviderConfig, Provider, providerFromConfig, getProviderTemplateReferences } from "../config/provider"
@@ -17,18 +18,36 @@ import { ProviderConfigContext } from "../config/config-context"
 import { ModuleConfig } from "../config/module"
 import { GardenPlugin } from "../types/plugin/plugin"
 import { joi } from "../config/common"
-import { validateWithPath } from "../config/validation"
+import { validateWithPath, validateSchema } from "../config/validation"
 import Bluebird from "bluebird"
-import { defaultEnvironmentStatus } from "../types/plugin/provider/getEnvironmentStatus"
+import { defaultEnvironmentStatus, EnvironmentStatus } from "../types/plugin/provider/getEnvironmentStatus"
 import { getPluginBases, getPluginBaseNames } from "../plugins"
 import { Profile } from "../util/profiling"
+import { join, dirname } from "path"
+import { readFile, writeFile, ensureDir } from "fs-extra"
+import { deserialize, serialize } from "v8"
+import { environmentStatusSchema } from "../config/status"
+import { hashString } from "../util/util"
 
 interface Params extends TaskParams {
   plugin: GardenPlugin
   config: ProviderConfig
+  forceRefresh: boolean
   forceInit: boolean
 }
 
+interface CachedStatus extends EnvironmentStatus {
+  configHash: string
+  resolvedAt: Date
+}
+
+const cachedStatusSchema = environmentStatusSchema().keys({
+  configHash: joi.string().required(),
+  resolvedAt: joi.date().required(),
+})
+
+const defaultCacheTtl = 3600 // 1 hour
+
 /**
  * Resolves the configuration for the specified provider.
  */
@@ -38,12 +57,14 @@ export class ResolveProviderTask extends BaseTask {
 
   private config: ProviderConfig
   private plugin: GardenPlugin
+  private forceRefresh: boolean
   private forceInit: boolean
 
   constructor(params: Params) {
     super(params)
     this.config = params.config
     this.plugin = params.plugin
+    this.forceRefresh = params.forceRefresh
     this.forceInit = params.forceInit
   }
 
@@ -96,6 +117,7 @@ export class ResolveProviderTask extends BaseTask {
             config,
             log: this.log,
             version: this.version,
+            forceRefresh: this.forceRefresh,
             forceInit: this.forceInit,
           })
         })
@@ -197,13 +219,86 @@ export class ResolveProviderTask extends BaseTask {
     return providerFromConfig(resolvedConfig, resolvedProviders, moduleConfigs, status)
   }
 
+  private getCachePath() {
+    return getProviderStatusCachePath({
+      gardenDirPath: this.garden.gardenDirPath,
+      pluginName: this.plugin.name,
+      environmentName: this.garden.environmentName,
+    })
+  }
+
+  private hashConfig(config: ProviderConfig) {
+    return hashString(stableStringify(config))
+  }
+
+  private async getCachedStatus(config: ProviderConfig): Promise<EnvironmentStatus | null> {
+    const cachePath = this.getCachePath()
+
+    this.log.silly(`Checking provider status cache for ${this.plugin.name} at ${cachePath}`)
+
+    let cachedStatus: CachedStatus | null = null
+
+    if (!this.forceRefresh) {
+      try {
+        const cachedData = deserialize(await readFile(cachePath))
+        cachedStatus = validateSchema(cachedData, cachedStatusSchema)
+      } catch (err) {
+        // Can't find or read a cached status
+        this.log.silly(`Unable to find or read provider status from ${cachePath}: ${err.message}`)
+      }
+    }
+
+    if (!cachedStatus) {
+      return null
+    }
+
+    const configHash = this.hashConfig(config)
+
+    if (cachedStatus.configHash !== configHash) {
+      this.log.silly(`Cached provider status at ${cachePath} does not match the current config`)
+      return null
+    }
+
+    const ttl = process.env.GARDEN_CACHE_TTL ? parseInt(process.env.GARDEN_CACHE_TTL, 10) : defaultCacheTtl
+    const cacheAge = (new Date().getTime() - cachedStatus?.resolvedAt.getTime()) / 1000
+
+    if (cacheAge > ttl) {
+      this.log.silly(`Cached provider status at ${cachePath} is out of date`)
+      return null
+    }
+
+    return omit(cachedStatus, ["configHash", "resolvedAt"])
+  }
+
+  private async setCachedStatus(config: ProviderConfig, status: EnvironmentStatus) {
+    const cachePath = this.getCachePath()
+    this.log.silly(`Caching provider status for ${this.plugin.name} at ${cachePath}`)
+
+    const cachedStatus: CachedStatus = {
+      ...status,
+      cached: true,
+      resolvedAt: new Date(),
+      configHash: this.hashConfig(config),
+    }
+
+    await ensureDir(dirname(cachePath))
+    await writeFile(cachePath, serialize(cachedStatus))
+  }
+
   private async ensurePrepared(tmpProvider: Provider) {
     const pluginName = tmpProvider.name
     const actions = await this.garden.getActionRouter()
     const ctx = this.garden.getPluginContext(tmpProvider)
 
     this.log.silly(`Getting status for ${pluginName}`)
 
+    // Check for cached provider status
+    const cachedStatus = await this.getCachedStatus(tmpProvider.config)
+
+    if (cachedStatus) {
+      return cachedStatus
+    }
+
     // TODO: avoid calling the handler manually (currently doing it to override the plugin context)
     const handler = await actions["getActionHandler"]({
       actionType: "getEnvironmentStatus",
@@ -245,6 +340,22 @@ export class ResolveProviderTask extends BaseTask {
       )
     }
 
+    if (!status.disableCache) {
+      await this.setCachedStatus(tmpProvider.config, status)
+    }
+
     return status
   }
 }
+
+export function getProviderStatusCachePath({
+  gardenDirPath,
+  pluginName,
+  environmentName,
+}: {
+  gardenDirPath: string
+  pluginName: string
+  environmentName: string
+}) {
+  return join(gardenDirPath, "cache", "provider-statuses", `${pluginName}.${environmentName}.json`)
+}

garden-service/src/types/plugin/provider/getEnvironmentStatus.ts

@@ -17,6 +17,8 @@ export interface EnvironmentStatus<O extends {} = any, D extends {} = any> {
   dashboardPages?: DashboardPage[]
   detail?: D
   outputs: O
+  disableCache?: boolean
+  cached?: boolean
 }
 
 export const defaultEnvironmentStatus: EnvironmentStatus = {

garden-service/src/util/util.ts

@@ -546,10 +546,11 @@ export function findByNames<T extends ObjectWithName>(names: string[], entries:
   return entries.filter(({ name }) => names.includes(name))
 }
 
-export function hashString(s: string, length: number) {
+export function hashString(s: string, length?: number): string {
   const urlHash = createHash("sha256")
   urlHash.update(s)
-  return urlHash.digest("hex").slice(0, length)
+  const str = urlHash.digest("hex")
+  return length ? str.slice(0, length) : str
 }
 
 /**

garden-service/src/vcs/vcs.ts

@@ -250,7 +250,7 @@ export function hashVersions(moduleConfig: ModuleConfig, versions: NamedTreeVers
   return hashStrings([configString, ...versionStrings])
 }
 
-export function hashStrings(hashes: string[]) {
+function hashStrings(hashes: string[]) {
   const versionHash = createHash("sha256")
   versionHash.update(hashes.join("."))
   return versionHash.digest("hex").slice(0, 10)