feat(workflows): add support for arbitrary user scripts in workflows

Users can now add script steps to their workflows. The usage is quite straightforward, just use a `script` key in your step instead of a `command`. The script is evaluated with `bash`, and should return an exit code of 0 if successful. I will follow up with a 2nd PR which introduces the ability to reference command outputs in script steps.
garden-io · Jun 24, 2020 · d7b76a4 · d7b76a4
1 parent 12e8b24
commit d7b76a4
Show file tree

Hide file tree

Showing 7 changed files with 345 additions and 60 deletions.
diff --git a/docs/reference/config.md b/docs/reference/config.md
@@ -934,11 +934,12 @@ description:
 # after these files are generated. This means you can reference the files specified here in your provider
 # configurations.
 files:
-  - # POSIX-style path to write the file to, relative to the project root. If the path contains one or more
-    # directories, they are created automatically if necessary.
+  - # POSIX-style path to write the file to, relative to the project root (or absolute). If the path contains one
+    # or more directories, they are created automatically if necessary.
     # If any of those directories conflict with existing file paths, or if the file path conflicts with an existing
     # directory path, an error will be thrown.
-    # Any existing file with the same path will be overwritten.
+    # **Any existing file with the same path will be overwritten, so be careful not to accidentally accidentally
+    # overwrite files unrelated to your workflow.**
     path:
 
     # The file data as a string.
@@ -960,7 +961,7 @@ limits:
 # The steps the workflow should run. At least one step is required. Steps are run sequentially. If a step fails,
 # subsequent steps are skipped.
 steps:
-  - # The Garden command this step should run.
+  - # A Garden command this step should run.
     #
     # Supported commands:
     #
@@ -984,6 +985,11 @@ steps:
     # A description of the workflow step.
     description:
 
+    # A bash script to run. Note that the host running the workflow must have bash installed and on path. It is
+    # considered to have run successfully if it returns an exit code of 0. Any other exit code signals an error, and
+    # the remainder of the workflow is aborted.
+    script:
+
 # A list of triggers that determine when the workflow should be run, and which environment should be used (Garden
 # Enterprise only).
 triggers:
@@ -1061,9 +1067,10 @@ Note that you cannot reference provider configuration in template strings within
 
 [files](#files) > path
 
-POSIX-style path to write the file to, relative to the project root. If the path contains one or more directories, they are created automatically if necessary.
+POSIX-style path to write the file to, relative to the project root (or absolute). If the path contains one
+or more directories, they are created automatically if necessary.
 If any of those directories conflict with existing file paths, or if the file path conflicts with an existing directory path, an error will be thrown.
-Any existing file with the same path will be overwritten.
+**Any existing file with the same path will be overwritten, so be careful not to accidentally accidentally overwrite files unrelated to your workflow.**
 
 | Type        | Required |
 | ----------- | -------- |
@@ -1142,7 +1149,7 @@ The steps the workflow should run. At least one step is required. Steps are run
 
 [steps](#steps) > command
 
-The Garden command this step should run.
+A Garden command this step should run.
 
 Supported commands:
 
@@ -1164,7 +1171,7 @@ Supported commands:
 
 | Type            | Required |
 | --------------- | -------- |
-| `array[string]` | Yes      |
+| `array[string]` | No       |
 
 ### `steps[].description`
 
@@ -1176,6 +1183,16 @@ A description of the workflow step.
 | -------- | -------- |
 | `string` | No       |
 
+### `steps[].script`
+
+[steps](#steps) > script
+
+A bash script to run. Note that the host running the workflow must have bash installed and on path. It is considered to have run successfully if it returns an exit code of 0. Any other exit code signals an error, and the remainder of the workflow is aborted.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
 ### `triggers[]`
 
 A list of triggers that determine when the workflow should be run, and which environment should be used (Garden Enterprise only).

diff --git a/garden-service/src/commands/run/workflow.ts b/garden-service/src/commands/run/workflow.ts
@@ -12,15 +12,18 @@ import { printHeader, getTerminalWidth, formatGardenError } from "../../logger/u
 import { StringParameter, Command, CommandParams, CommandResult, parseCliArgs } from "../base"
 import { dedent, wordWrap, deline } from "../../util/string"
 import { Garden } from "../../garden"
-import { getStepCommandConfigs, WorkflowFileSpec } from "../../config/workflow"
+import { getStepCommandConfigs, WorkflowStepSpec, WorkflowConfig, WorkflowFileSpec } from "../../config/workflow"
 import { LogEntry } from "../../logger/log-entry"
-import { GardenError } from "../../exceptions"
+import { GardenError, GardenBaseError } from "../../exceptions"
 import { WorkflowConfigContext } from "../../config/config-context"
 import { resolveTemplateStrings } from "../../template-string"
 import { ConfigurationError, FilesystemError } from "../../exceptions"
 import { posix, join } from "path"
 import { ensureDir, writeFile } from "fs-extra"
 import Bluebird from "bluebird"
+import { splitStream } from "../../util/util"
+import execa, { ExecaError } from "execa"
+import { LogLevel } from "../../logger/log-node"
 
 const runWorkflowArgs = {
   workflow: new StringParameter({
@@ -31,6 +34,10 @@ const runWorkflowArgs = {
 
 type Args = typeof runWorkflowArgs
 
+interface WorkflowRunOutput {
+  stepLogs: { [stepName: string]: string }
+}
+
 export class RunWorkflowCommand extends Command<Args, {}> {
   name = "workflow"
   help = "Run a workflow."
@@ -46,7 +53,13 @@ export class RunWorkflowCommand extends Command<Args, {}> {
 
   arguments = runWorkflowArgs
 
-  async action({ garden, log, headerLog, args, opts }: CommandParams<Args, {}>): Promise<CommandResult<null>> {
+  async action({
+    garden,
+    log,
+    headerLog,
+    args,
+    opts,
+  }: CommandParams<Args, {}>): Promise<CommandResult<WorkflowRunOutput>> {
     // Partially resolve the workflow config, and prepare any configured files before continuing
     const rawWorkflow = garden.getRawWorkflowConfig(args.workflow)
     const templateContext = new WorkflowConfigContext(garden, {}, garden.variables, garden.secrets)
@@ -63,50 +76,91 @@ export class RunWorkflowCommand extends Command<Args, {}> {
 
     const stepCommandConfigs = getStepCommandConfigs()
     const startedAt = new Date().valueOf()
+
+    const result = {
+      stepLogs: {},
+    }
+
     for (const [index, step] of steps.entries()) {
       printStepHeader(log, index, steps.length, step.description)
+
       const stepHeaderLog = log.placeholder({ indent: 1 })
       const stepBodyLog = log.placeholder({ indent: 1 })
       const stepFooterLog = log.placeholder({ indent: 1 })
-      let result: CommandResult
+      let commandResult: CommandResult
+      const inheritedOpts = cloneDeep(opts)
+
       try {
-        result = await runStepCommand({
-          commandSpec: step.command,
-          inheritedOpts: cloneDeep(opts),
-          garden,
-          headerLog: stepHeaderLog,
-          log: stepBodyLog,
-          footerLog: stepFooterLog,
-          stepCommandConfigs,
-        })
+        if (step.command) {
+          commandResult = await runStepCommand({
+            step,
+            inheritedOpts,
+            garden,
+            headerLog: stepHeaderLog,
+            log: stepBodyLog,
+            footerLog: stepFooterLog,
+            stepCommandConfigs,
+          })
+        } else if (step.script) {
+          commandResult = await runStepScript({
+            step,
+            inheritedOpts,
+            garden,
+            headerLog: stepHeaderLog,
+            log: stepBodyLog,
+            footerLog: stepFooterLog,
+          })
+        } else {
+          throw new ConfigurationError(`Workflow steps must specify either a command or a script.`, { step })
+        }
       } catch (err) {
-        throw err
+        printStepDuration({
+          log,
+          stepIndex: index,
+          stepCount: steps.length,
+          durationSecs: stepBodyLog.getDuration(),
+          success: false,
+        })
+        printResult({ startedAt, log, workflow, success: false })
+
+        logErrors(log, [err], index, steps.length, step.description)
+        return { result, errors: [err] }
       }
-      if (result.errors) {
-        logErrors(log, result.errors, index, steps.length, step.description)
-        return { errors: result.errors }
+
+      // Extract the text from the body log entry, info-level and higher
+      result.stepLogs[index.toString()] = stepBodyLog.toString((entry) => entry.level <= LogLevel.info)
+
+      if (commandResult.errors) {
+        logErrors(log, commandResult.errors, index, steps.length, step.description)
+        return { result, errors: commandResult.errors }
       }
-      printStepDuration(log, index, steps.length, stepBodyLog.getDuration())
+
+      printStepDuration({
+        log,
+        stepIndex: index,
+        stepCount: steps.length,
+        durationSecs: stepBodyLog.getDuration(),
+        success: true,
+      })
     }
-    const completedAt = new Date().valueOf()
-    const totalDuration = ((completedAt - startedAt) / 1000).toFixed(2)
 
-    log.info("")
-    log.info(chalk.magenta(`Workflow ${chalk.white(workflow.name)} completed.`))
-    log.info(chalk.magenta(`Total time elapsed: ${chalk.white(totalDuration)} Sec.`))
+    printResult({ startedAt, log, workflow, success: true })
 
-    return {}
+    return { result }
   }
 }
 
-export type RunStepCommandParams = {
+export interface RunStepParams {
   garden: Garden
   log: LogEntry
   headerLog: LogEntry
   footerLog: LogEntry
   inheritedOpts: any
+  step: WorkflowStepSpec
+}
+
+export interface RunStepCommandParams extends RunStepParams {
   stepCommandConfigs: any
-  commandSpec: string[]
 }
 
 export function printStepHeader(log: LogEntry, stepIndex: number, stepCount: number, stepDescription?: string) {
@@ -120,9 +174,23 @@ export function printStepHeader(log: LogEntry, stepIndex: number, stepCount: num
   log.info(header)
 }
 
-export function printStepDuration(log: LogEntry, stepIndex: number, stepCount: number, durationSecs: number) {
+export function printStepDuration({
+  log,
+  stepIndex,
+  stepCount,
+  durationSecs,
+  success,
+}: {
+  log: LogEntry
+  stepIndex: number
+  stepCount: number
+  durationSecs: number
+  success: boolean
+}) {
+  const result = success ? chalk.green("completed") : chalk.red("failed")
+
   const text = deline`
-    Step ${formattedStepNumber(stepIndex, stepCount)} ${chalk.green("completed")} in
+    Step ${formattedStepNumber(stepIndex, stepCount)} ${chalk.bold(result)} in
     ${chalk.white(durationSecs)} Sec
   `
   const maxWidth = Math.min(getTerminalWidth(), 120)
@@ -148,17 +216,38 @@ export function formattedStepNumber(stepIndex: number, stepCount: number) {
   return `${chalk.white(stepIndex + 1)}/${chalk.white(stepCount)}`
 }
 
+function printResult({
+  startedAt,
+  log,
+  workflow,
+  success,
+}: {
+  startedAt: number
+  log: LogEntry
+  workflow: WorkflowConfig
+  success: boolean
+}) {
+  const completedAt = new Date().valueOf()
+  const totalDuration = ((completedAt - startedAt) / 1000).toFixed(2)
+
+  const resultColor = success ? chalk.magenta : chalk.red
+
+  log.info("")
+  log.info(resultColor(`Workflow ${chalk.white(workflow.name)} completed.`))
+  log.info(chalk.magenta(`Total time elapsed: ${chalk.white(totalDuration)} Sec.`))
+}
+
 export async function runStepCommand({
   garden,
   log,
   footerLog,
   headerLog,
   inheritedOpts,
   stepCommandConfigs,
-  commandSpec,
+  step,
 }: RunStepCommandParams): Promise<CommandResult<any>> {
-  const config = stepCommandConfigs.find((c) => isEqual(c.prefix, take(commandSpec, c.prefix.length)))
-  const rest = commandSpec.slice(config.prefix.length) // arguments + options
+  const config = stepCommandConfigs.find((c) => isEqual(c.prefix, take(step.command!, c.prefix.length)))
+  const rest = step.command!.slice(config.prefix.length) // arguments + options
   const { args, opts } = parseCliArgs(rest, config.args, config.opts)
   const command: Command = new config.cmdClass()
   const result = await command.action({
@@ -226,3 +315,54 @@ async function writeWorkflowFile(garden: Garden, file: WorkflowFileSpec) {
     throw new FilesystemError(`Unable to write file '${file.path}': ${error.message}`, { error, file })
   }
 }
+
+class WorkflowScriptError extends GardenBaseError {
+  type = "workflow-script"
+}
+
+export async function runStepScript({ garden, log, step }: RunStepParams): Promise<CommandResult<any>> {
+  // Run the script, capturing any errors
+  const proc = execa("bash", ["-s"], {
+    all: true,
+    cwd: garden.projectRoot,
+    // The script is piped to stdin
+    input: step.script,
+    // Set a very large max buffer (we only hold one of these at a time, and want to avoid overflow errors)
+    buffer: true,
+    maxBuffer: 100 * 1024 * 1024,
+  })
+
+  // Stream output to `log`, splitting by line
+  const stdout = splitStream()
+  const stderr = splitStream()
+
+  stdout.on("error", () => {})
+  stdout.on("data", (line: Buffer) => {
+    log.info(line.toString())
+  })
+  stderr.on("error", () => {})
+  stderr.on("data", (line: Buffer) => {
+    log.info(line.toString())
+  })
+
+  proc.stdout!.pipe(stdout)
+  proc.stderr!.pipe(stderr)
+
+  try {
+    await proc
+    return {}
+  } catch (_err) {
+    const error = _err as ExecaError
+
+    // Unexpected error (failed to execute script, as opposed to script returning an error code)
+    if (!error.exitCode) {
+      throw error
+    }
+
+    throw new WorkflowScriptError(`Script exited with code ${error.exitCode}`, {
+      exitCode: error.exitCode,
+      stdout: error.stdout,
+      stderr: error.stderr,
+    })
+  }
+}