From 87ed678350f326ffcd9360dde37035aa4b138ede Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Wed, 11 Mar 2020 16:53:30 +0100 Subject: [PATCH] feat(core): pre-fetch provider tools and make tools a native feature This adds the notion of tools to the plugin framework, and a command to allow using and pre-fetching these tools. The primary motivation was to allow pre-fetching the tools as part of our container builds, as well as to make it easy for users to do the same in their custom CI setups with Garden. The `garden tools` command is also handy since it allows users to easily use the exact same tools as Garden's providers fetch and use. To pre-fetch the tools, we add a `garden util fetch-tools` command, which fetches all tools for the configured providers in the current project, or for all registered providers if the `--all` flag is set. Notes: - We now always fetch the docker binary on use, instead of checking for its existence locally. This avoids some code complexity and makes sure we always use the expected version. The downside should be mitigated by the pre-fetching in built containers, and the fetch-tools command. - We really need to reduce the number of tools and versions we bundle before we release 0.12. The container image sizes are too large atm. --- docs/reference/commands.md | 121 ++++-- garden-service/Dockerfile | 13 +- garden-service/buster.Dockerfile | 5 +- garden-service/src/actions.ts | 2 +- garden-service/src/analytics/analytics.ts | 10 +- garden-service/src/cli/cli.ts | 5 +- garden-service/src/commands/commands.ts | 10 +- garden-service/src/commands/plugins.ts | 2 +- garden-service/src/commands/tools.ts | 212 +++++++++++ garden-service/src/commands/util.ts | 104 +++++ garden-service/src/config/config-context.ts | 17 +- garden-service/src/config/provider.ts | 22 +- garden-service/src/config/workflow.ts | 4 +- garden-service/src/garden.ts | 31 +- garden-service/src/logger/writers/base.ts | 2 +- .../logger/writers/basic-terminal-writer.ts | 2 +- garden-service/src/outputs.ts | 2 +- garden-service/src/plugins.ts | 10 + .../src/plugins/conftest/conftest.ts | 70 ++-- garden-service/src/plugins/container/build.ts | 19 +- .../src/plugins/container/container.ts | 44 +++ .../src/plugins/container/helpers.ts | 111 ++---- .../src/plugins/container/publish.ts | 13 +- .../src/plugins/hadolint/hadolint.ts | 46 ++- garden-service/src/plugins/kubernetes/api.ts | 2 +- .../commands/cleanup-cluster-registry.ts | 3 +- .../plugins/kubernetes/commands/pull-image.ts | 38 +- .../src/plugins/kubernetes/container/build.ts | 24 +- .../src/plugins/kubernetes/container/exec.ts | 3 +- .../plugins/kubernetes/container/publish.ts | 18 +- .../src/plugins/kubernetes/helm/helm-cli.ts | 92 +++-- .../src/plugins/kubernetes/helm/tiller.ts | 4 +- .../src/plugins/kubernetes/hot-reload.ts | 3 +- .../kubernetes/integrations/cert-manager.ts | 2 +- .../src/plugins/kubernetes/kubectl.ts | 57 ++- .../src/plugins/kubernetes/kubernetes.ts | 9 +- .../src/plugins/kubernetes/local/config.ts | 5 +- .../src/plugins/kubernetes/local/microk8s.ts | 10 +- garden-service/src/plugins/kubernetes/logs.ts | 34 +- .../src/plugins/kubernetes/namespace.ts | 2 +- .../src/plugins/kubernetes/port-forward.ts | 2 +- garden-service/src/plugins/kubernetes/run.ts | 12 +- .../maven-container/maven-container.ts | 10 +- .../src/plugins/maven-container/maven.ts | 34 +- .../src/plugins/maven-container/openjdk.ts | 91 +++-- garden-service/src/plugins/openfaas/build.ts | 3 +- garden-service/src/plugins/openfaas/config.ts | 9 +- .../src/plugins/openfaas/faas-cli.ts | 24 +- .../src/plugins/openfaas/openfaas.ts | 11 +- garden-service/src/plugins/terraform/cli.ts | 173 ++++++--- .../src/plugins/terraform/commands.ts | 4 +- .../src/plugins/terraform/common.ts | 21 +- garden-service/src/plugins/terraform/init.ts | 10 +- .../src/plugins/terraform/module.ts | 6 +- .../src/plugins/terraform/terraform.ts | 3 +- garden-service/src/process.ts | 12 +- garden-service/src/tasks/resolve-module.ts | 10 +- garden-service/src/tasks/resolve-provider.ts | 30 +- garden-service/src/types/plugin/plugin.ts | 15 + .../src/types/plugin/provider/augmentGraph.ts | 8 +- .../plugin/provider/configureProvider.ts | 15 +- garden-service/src/types/plugin/tools.ts | 98 +++++ garden-service/src/util/ext-tools.ts | 359 ++++++++---------- garden-service/src/util/util.ts | 19 + garden-service/test/data/tools/tool-a.sh | 3 + garden-service/test/data/tools/tool-b.sh | 3 + .../integ/src/plugins/container/helpers.ts | 34 +- .../plugins/kubernetes/commands/pull-image.ts | 22 +- .../src/plugins/kubernetes/container/build.ts | 17 +- garden-service/test/unit/src/actions.ts | 3 +- .../test/unit/src/commands/plugins.ts | 2 +- .../test/unit/src/commands/tools.ts | 290 ++++++++++++++ garden-service/test/unit/src/commands/util.ts | 134 +++++++ .../test/unit/src/config/config-context.ts | 6 +- .../test/unit/src/config/workflow.ts | 4 +- garden-service/test/unit/src/garden.ts | 144 ++++--- .../unit/src/plugins/container/container.ts | 91 ++--- .../plugins/kubernetes/container/ingress.ts | 75 ++-- .../test/unit/src/plugins/kubernetes/init.ts | 3 +- .../unit/src/plugins/kubernetes/kubernetes.ts | 16 +- 80 files changed, 2123 insertions(+), 886 deletions(-) create mode 100644 garden-service/src/commands/tools.ts create mode 100644 garden-service/src/commands/util.ts create mode 100644 garden-service/src/types/plugin/tools.ts create mode 100644 garden-service/test/data/tools/tool-a.sh create mode 100644 garden-service/test/data/tools/tool-b.sh create mode 100644 garden-service/test/unit/src/commands/tools.ts create mode 100644 garden-service/test/unit/src/commands/util.ts diff --git a/docs/reference/commands.md b/docs/reference/commands.md index b064fda2bc7..db813f089dd 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -86,6 +86,34 @@ Note: Currently only supports simple GET requests for HTTP/HTTPS ingresses. | -------- | -------- | ----------- | | `serviceAndPath` | Yes | The name of the service to call followed by the ingress path (e.g. my-container/somepath). +### garden config analytics-enabled + +Update your preferences regarding analytics. + +To help us make Garden better, you can opt in to the collection of usage data. +We make sure all the data collected is anonymized and stripped of sensitive +information. We collect data about which commands are run, what tasks they trigger, +which API calls are made to your local Garden server, as well as some info +about the environment in which Garden runs. + +You will be asked if you want to opt-in when running Garden for the +first time and you can use this command to update your preferences later. + +Examples: + + garden config analytics-enabled true # enable analytics + garden config analytics-enabled false # disable analytics + +##### Usage + + garden config analytics-enabled [enable] + +##### Arguments + +| Argument | Required | Description | +| -------- | -------- | ----------- | + | `enable` | No | Enable analytics. Defaults to "true" + ### garden create project Create a new Garden project. @@ -793,6 +821,56 @@ Examples: | `--force-build` | | boolean | Force rebuild of module(s). | `--watch` | `-w` | boolean | Watch for changes in module(s) and auto-test. +### garden tools + +Access tools included by providers. + +Run a tool defined by a provider in your project, downloading and extracting it if +necessary. Run without arguments to get a list of all tools available. + +Run with the --get-path flag to just print the path to the binary or library +directory (depending on the tool type). If the tool is a non-executable library, this +flag is implicit. + +When multiple plugins provide a tool with the same name, you can choose a specific +plugin/version by specifying ., instead of just . +This is generally advisable when using this command in scripts, to avoid accidental +conflicts. + +When there are name conflicts and a plugin name is not specified, the preference is +for defined by configured providers in the current project (if applicable), and then +alphabetical by plugin name. + +Examples: + + # Run kubectl with . + garden tools kubectl -- + + # Run the kubectl version defined specifically by the `kubernetes` plugin. + garden tools kubernetes.kubectl -- + + # Print the path to the kubernetes.kubectl tool to stdout, instead of running it. + garden tools kubernetes.kubectl --get-path + + # List all available tools. + garden tools + +##### Usage + + garden tools [tool] [options] + +##### Arguments + +| Argument | Required | Description | +| -------- | -------- | ----------- | + | `tool` | No | The name of the tool to run. + +##### Options + +| Argument | Alias | Type | Description | +| -------- | ----- | ---- | ----------- | + | `--get-path` | | boolean | If specified, we print the path to the binary or library instead of running it. + ### garden unlink source Unlink a previously linked remote source from its local directory. @@ -904,41 +982,36 @@ Examples: garden update-remote all -### garden validate +### garden util fetch-tools -Check your garden configuration for errors. +Pre-fetch plugin tools. -Throws an error and exits with code 1 if something's not right in your garden.yml files. +Pre-fetch all the available tools for the configured providers in the current +project/environment, or all registered providers if the --all parameter is +specified. -##### Usage +Examples: - garden validate + garden util fetch-tools # fetch for just the current project/env + garden util fetch-tools --all # fetch for all registered providers -### garden config analytics-enabled +##### Usage -Update your preferences regarding analytics. + garden util fetch-tools [options] -To help us make Garden better, you can opt in to the collection of usage data. -We make sure all the data collected is anonymized and stripped of sensitive -information. We collect data about which commands are run, what tasks they trigger, -which API calls are made to your local Garden server, as well as some info -about the environment in which Garden runs. - -You will be asked if you want to opt-in when running Garden for the -first time and you can use this command to update your preferences later. +##### Options -Examples: +| Argument | Alias | Type | Description | +| -------- | ----- | ---- | ----------- | + | `--all` | | boolean | Fetch all tools for registered plugins, instead of just ones in the current env/project. - garden config analytics-enabled true # enable analytics - garden config analytics-enabled false # disable analytics +### garden validate -##### Usage +Check your garden configuration for errors. - garden config analytics-enabled [enable] +Throws an error and exits with code 1 if something's not right in your garden.yml files. -##### Arguments +##### Usage -| Argument | Required | Description | -| -------- | -------- | ----------- | - | `enable` | No | Enable analytics. Defaults to "true" + garden validate diff --git a/garden-service/Dockerfile b/garden-service/Dockerfile index b2545790c3b..1e775bae804 100644 --- a/garden-service/Dockerfile +++ b/garden-service/Dockerfile @@ -11,17 +11,17 @@ RUN apk add --no-cache \ WORKDIR /tmp -RUN npm install pkg@4.4.2 && node_modules/.bin/pkg-fetch node12 alpine x64 +RUN npm install pkg@4.4.8 && node_modules/.bin/pkg-fetch node12 alpine x64 ADD package.json /tmp/ ADD package-lock.json /tmp/ RUN npm install \ && rm -rf /root/.npm/* \ - /usr/lib/node_modules/npm/man/* \ - /usr/lib/node_modules/npm/doc/* \ - /usr/lib/node_modules/npm/html/* \ - /usr/lib/node_modules/npm/scripts/* + /usr/lib/node_modules/npm/man/* \ + /usr/lib/node_modules/npm/doc/* \ + /usr/lib/node_modules/npm/html/* \ + /usr/lib/node_modules/npm/scripts/* ADD bin/garden /tmp/bin/garden ADD bin/garden-debug /tmp/bin/garden-debug @@ -56,6 +56,7 @@ WORKDIR /project RUN chmod +x /garden/garden \ && ln -s /garden/garden /bin/garden \ - && chmod +x /bin/garden + && chmod +x /bin/garden \ + && cd /garden/static && garden util fetch-tools --all --logger-type=basic ENTRYPOINT ["/garden/garden"] diff --git a/garden-service/buster.Dockerfile b/garden-service/buster.Dockerfile index 4aa346dcae5..3f0873908ce 100644 --- a/garden-service/buster.Dockerfile +++ b/garden-service/buster.Dockerfile @@ -26,6 +26,9 @@ ADD . /garden WORKDIR /project RUN ln -s /garden/garden /bin/garden \ - && chmod +x /bin/garden + && chmod +x /bin/garden \ + && cd /garden/static \ + && git init \ + && garden util fetch-tools --all --logger-type=basic ENTRYPOINT ["/garden/garden"] diff --git a/garden-service/src/actions.ts b/garden-service/src/actions.ts index 2b0d349b07f..c04bbb345c7 100644 --- a/garden-service/src/actions.ts +++ b/garden-service/src/actions.ts @@ -624,7 +624,7 @@ export class ActionRouter implements TypeGuard { const environmentStatuses: EnvironmentStatusMap = {} const providers = await this.garden.resolveProviders() - await Bluebird.each(providers, async (provider) => { + await Bluebird.each(Object.values(providers), async (provider) => { await this.cleanupEnvironment({ pluginName: provider.name, log: envLog }) environmentStatuses[provider.name] = await this.getEnvironmentStatus({ pluginName: provider.name, log: envLog }) }) diff --git a/garden-service/src/analytics/analytics.ts b/garden-service/src/analytics/analytics.ts index f58546e5f04..a3a9c2b276a 100644 --- a/garden-service/src/analytics/analytics.ts +++ b/garden-service/src/analytics/analytics.ts @@ -19,7 +19,6 @@ import { Garden } from "../garden" import { AnalyticsType } from "./analytics-types" import dedent from "dedent" import { getGitHubUrl } from "../docs/common" -import { InternalError } from "../exceptions" const API_KEY = process.env.ANALYTICS_DEV ? SEGMENT_DEV_API_KEY : SEGMENT_PROD_API_KEY @@ -126,15 +125,12 @@ export class AnalyticsHandler { private ciName = ci.name private systemConfig: SystemInfo private isCI = ci.isCI - private sessionId: string + private sessionId: string | null private pendingEvents: Map protected garden: Garden private projectMetadata: ProjectMetadata private constructor(garden: Garden, log: LogEntry) { - if (!garden.sessionId) { - throw new InternalError(`Garden instance with null sessionId passed to AnalyticsHandler constructor.`, {}) - } this.segment = new segmentClient(API_KEY, { flushAt: 20, flushInterval: 300 }) this.log = log this.garden = garden @@ -247,7 +243,7 @@ export class AnalyticsHandler { * Used internally to check if a users has opted-in or not. */ private analyticsEnabled(): boolean { - if (process.env.GARDEN_DISABLE_ANALYTICS === "true") { + if (!this.sessionId || process.env.GARDEN_DISABLE_ANALYTICS === "true") { return false } return this.analyticsConfig.optedIn || false @@ -282,7 +278,7 @@ export class AnalyticsHandler { ciName: this.ciName, system: this.systemConfig, isCI: this.isCI, - sessionId: this.sessionId, + sessionId: this.sessionId!, projectMetadata: this.projectMetadata, } } diff --git a/garden-service/src/cli/cli.ts b/garden-service/src/cli/cli.ts index 37cd8ddfe76..f69a63f59bf 100644 --- a/garden-service/src/cli/cli.ts +++ b/garden-service/src/cli/cli.ts @@ -73,9 +73,9 @@ const GLOBAL_OPTIONS_GROUP_NAME = "Global options" * Used by commands that have noProject=true. That is, commands that need * to run outside of valid Garden projects. */ -class DummyGarden extends Garden { +export class DummyGarden extends Garden { async resolveProviders() { - return [] + return {} } async scanAndAddConfigs() {} } @@ -307,7 +307,6 @@ export class GardenCli { // the screen the logs are printed. const headerLog = logger.placeholder() const log = logger.placeholder() - logger.info("") const footerLog = logger.placeholder() // Init event & log streaming. diff --git a/garden-service/src/commands/commands.ts b/garden-service/src/commands/commands.ts index 329901200f1..f9ce163229e 100644 --- a/garden-service/src/commands/commands.ts +++ b/garden-service/src/commands/commands.ts @@ -32,10 +32,13 @@ import { ConfigCommand } from "./config/config" import { PluginsCommand } from "./plugins" import { LoginCommand } from "./login" import { LogOutCommand } from "./logout" +import { ToolsCommand } from "./tools" +import { UtilCommand } from "./util" export const coreCommands: Command[] = [ new BuildCommand(), new CallCommand(), + new ConfigCommand(), new CreateCommand(), new DeleteCommand(), new DeployCommand(), @@ -43,6 +46,8 @@ export const coreCommands: Command[] = [ new ExecCommand(), new GetCommand(), new LinkCommand(), + new LoginCommand(), + new LogOutCommand(), new LogsCommand(), new MigrateCommand(), new OptionsCommand(), @@ -53,10 +58,9 @@ export const coreCommands: Command[] = [ new ServeCommand(), new SetCommand(), new TestCommand(), + new ToolsCommand(), new UnlinkCommand(), new UpdateRemoteCommand(), + new UtilCommand(), new ValidateCommand(), - new ConfigCommand(), - new LoginCommand(), - new LogOutCommand(), ] diff --git a/garden-service/src/commands/plugins.ts b/garden-service/src/commands/plugins.ts index ee1c6db816d..cb8023d5b53 100644 --- a/garden-service/src/commands/plugins.ts +++ b/garden-service/src/commands/plugins.ts @@ -127,7 +127,7 @@ async function listPlugins(garden: Garden, log: LogEntry, pluginsToList: string[ log.info(dedent` ${chalk.white.bold("USAGE")} - garden ${chalk.yellow("[global options]")} ${chalk.blueBright("")} ${chalk.white("[args ...]")} + garden ${chalk.yellow("[global options]")} ${chalk.blueBright("")} -- ${chalk.white("[args ...]")} ${chalk.white.bold("PLUGIN COMMANDS")} `) diff --git a/garden-service/src/commands/tools.ts b/garden-service/src/commands/tools.ts new file mode 100644 index 00000000000..491f858ef6c --- /dev/null +++ b/garden-service/src/commands/tools.ts @@ -0,0 +1,212 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import chalk from "chalk" +import { max, omit, sortBy } from "lodash" +import { dedent, renderTable, tablePresets } from "../util/string" +import { LogEntry } from "../logger/log-entry" +import { Garden } from "../garden" +import { Command, CommandParams, StringOption, BooleanParameter } from "./base" +import { getTerminalWidth } from "../logger/util" +import { LoggerType } from "../logger/logger" +import { ParameterError } from "../exceptions" +import { uniqByName, exec } from "../util/util" +import { PluginTool } from "../util/ext-tools" +import { findProjectConfig } from "../config/base" +import { DummyGarden } from "../cli/cli" + +const toolsArgs = { + tool: new StringOption({ + help: "The name of the tool to run.", + required: false, + }), +} + +const toolsOpts = { + "get-path": new BooleanParameter({ + help: "If specified, we print the path to the binary or library instead of running it.", + required: false, + }), +} + +type Args = typeof toolsArgs +type Opts = typeof toolsOpts + +export class ToolsCommand extends Command { + name = "tools" + help = "Access tools included by providers." + cliOnly = true + + // FIXME: We need this while we're still resolving providers in the AnalyticsHandler + noProject = true + + description = dedent` + Run a tool defined by a provider in your project, downloading and extracting it if + necessary. Run without arguments to get a list of all tools available. + + Run with the --get-path flag to just print the path to the binary or library + directory (depending on the tool type). If the tool is a non-executable library, this + flag is implicit. + + When multiple plugins provide a tool with the same name, you can choose a specific + plugin/version by specifying ., instead of just . + This is generally advisable when using this command in scripts, to avoid accidental + conflicts. + + When there are name conflicts and a plugin name is not specified, the preference is + for defined by configured providers in the current project (if applicable), and then + alphabetical by plugin name. + + Examples: + + # Run kubectl with . + garden tools kubectl -- + + # Run the kubectl version defined specifically by the \`kubernetes\` plugin. + garden tools kubernetes.kubectl -- + + # Print the path to the kubernetes.kubectl tool to stdout, instead of running it. + garden tools kubernetes.kubectl --get-path + + # List all available tools. + garden tools + ` + + arguments = toolsArgs + options = toolsOpts + + getLoggerType(): LoggerType { + return "basic" + } + + async prepare({ log }) { + // Override the logger output, to output to stderr instead of stdout, to avoid contaminating command output + log.root.writers.find((w) => w.type === "basic")!.output = process.stderr + return { persistent: false } + } + + async action({ garden, log, args, opts }: CommandParams) { + if (!args.tool) { + // We're listing tools, not executing one + return printTools(garden, log) + } + + let pluginName: string | null = null + let toolName: string + + const split = args.tool.split(".") + + if (split.length === 1) { + toolName = args.tool + } else if (split.length === 2) { + pluginName = split[0] + toolName = split[1] + } else { + throw new ParameterError( + `Invalid tool name argument. Please specify either a tool name (no periods) or ..`, + { args } + ) + } + + // We're executing a tool + const availablePlugins = Object.values(garden["registeredPlugins"]) + let plugins = availablePlugins + + if (pluginName) { + plugins = plugins.filter((p) => p.name === pluginName) + + if (plugins.length === 0) { + throw new ParameterError(`Could not find plugin ${pluginName}.`, { availablePlugins }) + } + } else { + // Place configured providers at the top for preference, if applicable + const projectRoot = await findProjectConfig(garden.projectRoot) + + if (projectRoot) { + if (garden instanceof DummyGarden) { + garden = await Garden.factory(garden.projectRoot, { ...omit(garden.opts, "config"), log }) + } + const configuredPlugins = await garden.getPlugins() + plugins = uniqByName([...configuredPlugins, ...availablePlugins]) + } + } + + const matchedTools = sortBy(plugins, "name") + .flatMap((plugin) => (plugin.tools || []).map((tool) => ({ plugin, tool }))) + .filter(({ tool }) => tool.name === toolName) + + const matchedNames = matchedTools.map(({ plugin, tool }) => `${plugin.name}.${tool.name}`) + + if (matchedTools.length === 0) { + throw new ParameterError(`Could not find tool ${args.tool}.`, { args }) + } + + if (matchedTools.length > 1) { + log.warn(chalk.yellow(`Multiple tools matched (${matchedNames.join(", ")}). Running ${matchedNames[0]}`)) + } + + const toolCls = new PluginTool(matchedTools[0].tool) + const path = await toolCls.getPath(log) + + // We just output the path if --get-path is set, or if the tool is a library + if (opts["get-path"] || toolCls.type === "library") { + process.stdout.write(path + "\n") + return { path } + } + + // We're running a binary + if (opts.output) { + // We collect the output and return + const result = await exec(path, args._ || [], { reject: false }) + return { path, stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode } + } else { + // We attach stdout and stderr directly, and exit with the same code as we get from the command + log.stop() + const result = await exec(path, args._ || [], { reject: false, stdio: "inherit" }) + process.exit(result.exitCode) + } + } +} + +async function printTools(garden: Garden, log: LogEntry) { + log.info(dedent` + ${chalk.white.bold("USAGE")} + + garden ${chalk.yellow("[global options]")} ${chalk.blueBright("")} -- ${chalk.white("[args ...]")} + garden ${chalk.yellow("[global options]")} ${chalk.blueBright("")} --get-path + + ${chalk.white.bold("PLUGIN TOOLS")} + `) + + const registeredPlugins = Object.values(garden["registeredPlugins"]) + + const allTools = sortBy(registeredPlugins, "name").flatMap((plugin) => + (plugin.tools || []).map((tool) => ({ plugin, tool })) + ) + + const rows = allTools.map(({ plugin, tool }) => { + return [ + ` ${chalk.cyan(plugin.name + ".")}${chalk.cyan.bold(tool.name)}`, + chalk.gray(`[${tool.type}]`), + tool.description, + ] + }) + + const maxRowLength = max(rows.map((r) => r[0].length))! + + log.info( + renderTable(rows, { + ...tablePresets["no-borders"], + colWidths: [null, null, getTerminalWidth() - maxRowLength - 2], + }) + ) + + log.info("") + + return {} +} diff --git a/garden-service/src/commands/util.ts b/garden-service/src/commands/util.ts new file mode 100644 index 00000000000..9457740d2cb --- /dev/null +++ b/garden-service/src/commands/util.ts @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { Command, CommandParams, BooleanParameter } from "./base" +import { RuntimeError } from "../exceptions" +import dedent from "dedent" +import { GardenPlugin } from "../types/plugin/plugin" +import { findProjectConfig } from "../config/base" +import { Garden } from "../garden" +import Bluebird from "bluebird" +import { PluginTool } from "../util/ext-tools" +import { fromPairs, omit, uniqBy } from "lodash" +import { printHeader, printFooter } from "../logger/util" +import { DummyGarden } from "../cli/cli" + +export class UtilCommand extends Command { + name = "util" + help = "Misc utility commands." + + subCommands = [FetchToolsCommand] + + async action() { + return {} + } +} + +const fetchToolsOpts = { + all: new BooleanParameter({ + help: "Fetch all tools for registered plugins, instead of just ones in the current env/project.", + required: false, + }), +} + +type FetchToolsOpts = typeof fetchToolsOpts + +export class FetchToolsCommand extends Command<{}, FetchToolsOpts> { + name = "fetch-tools" + help = "Pre-fetch plugin tools." + cliOnly = true + + noProject = true + + description = dedent` + Pre-fetch all the available tools for the configured providers in the current + project/environment, or all registered providers if the --all parameter is + specified. + + Examples: + + garden util fetch-tools # fetch for just the current project/env + garden util fetch-tools --all # fetch for all registered providers + ` + + options = fetchToolsOpts + + async action({ garden, log, opts }: CommandParams<{}, FetchToolsOpts>) { + let plugins: GardenPlugin[] + + if (opts.all) { + plugins = Object.values(garden.registeredPlugins) + printHeader(log, "Fetching tools for all registered providers", "hammer_and_wrench") + } else { + const projectRoot = findProjectConfig(garden.projectRoot) + + if (!projectRoot) { + throw new RuntimeError( + `Could not find project config in the current directory, or anywhere above. Please use the --all parameter if you'd like to fetch tools for all registered providers.`, + { root: garden.projectRoot } + ) + } + + if (garden instanceof DummyGarden) { + garden = await Garden.factory(garden.projectRoot, { ...omit(garden.opts, "config"), log }) + } + + plugins = await garden.getPlugins() + + printHeader(log, "Fetching all tools for the current project and environment", "hammer_and_wrench") + } + + const tools = plugins.flatMap((plugin) => + (plugin.tools || []).map((spec) => ({ plugin, tool: new PluginTool(spec) })) + ) + // No need to fetch the same tools multiple times, if they're used in multiple providers + const deduplicated = uniqBy(tools, ({ tool }) => tool["versionPath"]) + + const paths = fromPairs( + await Bluebird.map(deduplicated, async ({ plugin, tool }) => { + const fullName = `${plugin.name}.${tool.name}` + const path = await tool.getPath(log) + return [fullName, { type: tool.type, path }] + }) + ) + + printFooter(log) + + return { result: paths } + } +} diff --git a/garden-service/src/config/config-context.ts b/garden-service/src/config/config-context.ts index d5caacdc1ce..efe4c19a9ca 100644 --- a/garden-service/src/config/config-context.ts +++ b/garden-service/src/config/config-context.ts @@ -8,9 +8,9 @@ import Joi from "@hapi/joi" import chalk from "chalk" -import { isString, fromPairs } from "lodash" +import { isString, fromPairs, mapValues } from "lodash" import { PrimitiveMap, joiIdentifierMap, joiStringMap, joiPrimitive, DeepPrimitiveMap, joiVariables } from "./common" -import { Provider, ProviderConfig } from "./provider" +import { Provider, ProviderConfig, ProviderMap } from "./provider" import { ConfigurationError } from "../exceptions" import { resolveTemplateString } from "../template-string" import { Garden } from "../garden" @@ -396,8 +396,9 @@ export class ProviderConfigContext extends ProjectConfigContext { ) public secrets: PrimitiveMap - constructor(garden: Garden, resolvedProviders: Provider[], variables: DeepPrimitiveMap, secrets: PrimitiveMap) { + constructor(garden: Garden, resolvedProviders: ProviderMap, variables: DeepPrimitiveMap, secrets: PrimitiveMap) { super({ projectName: garden.projectName, artifactsPath: garden.artifactsPath, username: garden.username }) + const _this = this const fullEnvName = garden.namespace ? `${garden.namespace}.${garden.environmentName}` : garden.environmentName @@ -405,9 +406,7 @@ export class ProviderConfigContext extends ProjectConfigContext { this.project = new ProjectContext(this, garden.projectName) - this.providers = new Map( - resolvedProviders.map((p) => <[string, ProviderContext]>[p.name, new ProviderContext(_this, p)]) - ) + this.providers = new Map(Object.entries(mapValues(resolvedProviders, (p) => new ProviderContext(_this, p)))) this.var = this.variables = variables this.secrets = secrets @@ -620,7 +619,7 @@ export class ModuleConfigContext extends ProviderConfigContext { runtimeContext, }: { garden: Garden - resolvedProviders: Provider[] + resolvedProviders: ProviderMap variables: DeepPrimitiveMap secrets: PrimitiveMap moduleName?: string @@ -660,7 +659,7 @@ export class OutputConfigContext extends ModuleConfigContext { runtimeContext, }: { garden: Garden - resolvedProviders: Provider[] + resolvedProviders: ProviderMap variables: DeepPrimitiveMap secrets: PrimitiveMap modules: Module[] @@ -680,7 +679,7 @@ export class OutputConfigContext extends ModuleConfigContext { } export class WorkflowConfigContext extends ProviderConfigContext { - constructor(garden: Garden, resolvedProviders: Provider[], variables: DeepPrimitiveMap, secrets: PrimitiveMap) { + constructor(garden: Garden, resolvedProviders: ProviderMap, variables: DeepPrimitiveMap, secrets: PrimitiveMap) { super(garden, resolvedProviders, variables, secrets) } } diff --git a/garden-service/src/config/provider.ts b/garden-service/src/config/provider.ts index 7afdcf99f74..b29d4125b16 100644 --- a/garden-service/src/config/provider.ts +++ b/garden-service/src/config/provider.ts @@ -7,7 +7,7 @@ */ import { deline } from "../util/string" -import { joiIdentifier, joiUserIdentifier, joiArray, joi } from "./common" +import { joiIdentifier, joiUserIdentifier, joiArray, joi, joiIdentifierMap } from "./common" import { collectTemplateReferences } from "../template-string" import { ConfigurationError } from "../exceptions" import { ModuleConfig, moduleConfigSchema } from "./module" @@ -15,6 +15,7 @@ import { uniq } from "lodash" import { GardenPlugin } from "../types/plugin/plugin" import { EnvironmentStatus } from "../types/plugin/provider/getEnvironmentStatus" import { environmentStatusSchema } from "./status" +import { PluginTools } from "../types/plugin/tools" export interface ProviderConfig { name: string @@ -49,22 +50,26 @@ export const providerConfigBaseSchema = () => export interface Provider { name: string - dependencies: Provider[] + dependencies: { [name: string]: Provider } environments?: string[] moduleConfigs: ModuleConfig[] config: T status: EnvironmentStatus + tools: PluginTools } export const providerSchema = () => providerFixedFieldsSchema() .keys({ - dependencies: joiArray(joi.link("...")) - .description("List of all the providers that this provider depends on.") + dependencies: joiIdentifierMap(joi.link("...")) + .description("Map of all the providers that this provider depends on.") .required(), config: providerConfigBaseSchema().required(), moduleConfigs: joiArray(moduleConfigSchema().optional()), status: environmentStatusSchema(), + tools: joiIdentifierMap(joi.object()) + .required() + .description("Map of tools defined by the provider."), }) .id("provider") @@ -77,17 +82,19 @@ export const defaultProviders = [{ name: "container" }] // this is used for default handlers in the action handler export const defaultProvider: Provider = { name: "_default", - dependencies: [], + dependencies: {}, moduleConfigs: [], config: { name: "_default" }, status: { ready: true, outputs: {} }, + tools: {}, } export function providerFromConfig( config: ProviderConfig, - dependencies: Provider[], + dependencies: ProviderMap, moduleConfigs: ModuleConfig[], - status: EnvironmentStatus + status: EnvironmentStatus, + tools: PluginTools ): Provider { return { name: config.name, @@ -95,6 +102,7 @@ export function providerFromConfig( moduleConfigs, config, status, + tools, } } diff --git a/garden-service/src/config/workflow.ts b/garden-service/src/config/workflow.ts index 58010987d58..ab32e18df64 100644 --- a/garden-service/src/config/workflow.ts +++ b/garden-service/src/config/workflow.ts @@ -12,7 +12,7 @@ import { DEFAULT_API_VERSION } from "../constants" import { deline, dedent } from "../util/string" import { defaultContainerLimits, ServiceLimitSpec } from "../plugins/container/config" import { Garden } from "../garden" -import { Provider } from "./provider" +import { ProviderMap } from "./provider" import { WorkflowConfigContext } from "./config-context" import { resolveTemplateStrings } from "../template-string" import { validateWithPath } from "./validation" @@ -179,7 +179,7 @@ export interface WorkflowConfigMap { [key: string]: WorkflowConfig } -export function resolveWorkflowConfig(garden: Garden, resolvedProviders: Provider[], config: WorkflowConfig) { +export function resolveWorkflowConfig(garden: Garden, resolvedProviders: ProviderMap, config: WorkflowConfig) { const log = garden.log const { variables, secrets } = garden const context = new WorkflowConfigContext(garden, resolvedProviders, variables, secrets) diff --git a/garden-service/src/garden.ts b/garden-service/src/garden.ts index cd39ced2738..4bd60a0aa6d 100644 --- a/garden-service/src/garden.ts +++ b/garden-service/src/garden.ts @@ -12,7 +12,7 @@ import { ensureDir } from "fs-extra" import dedent from "dedent" import { platform, arch } from "os" import { parse, relative, resolve, dirname } from "path" -import { flatten, isString, sortBy, fromPairs, keyBy } from "lodash" +import { flatten, isString, sortBy, fromPairs, keyBy, mapValues, omit } from "lodash" const AsyncLock = require("async-lock") import { TreeCache } from "./cache" @@ -52,7 +52,13 @@ import { detectModuleOverlap, ModuleOverlap, } from "./util/fs" -import { Provider, ProviderConfig, getAllProviderDependencyNames, defaultProvider } from "./config/provider" +import { + Provider, + ProviderConfig, + getAllProviderDependencyNames, + defaultProvider, + ProviderMap, +} from "./config/provider" import { ResolveProviderTask } from "./tasks/resolve-provider" import { ActionRouter } from "./actions" import { RuntimeContext } from "./runtime-context" @@ -141,7 +147,7 @@ export class Garden { private pluginModuleConfigs: ModuleConfig[] private resolvedProviders: { [key: string]: Provider } protected configsScanned: boolean - private readonly registeredPlugins: { [key: string]: GardenPlugin } + public readonly registeredPlugins: { [key: string]: GardenPlugin } private readonly taskGraph: TaskGraph private watcher: Watcher private asyncLock: any @@ -259,6 +265,7 @@ export class Garden { opts: GardenOpts = {} ): Promise> { let { environmentName: environmentStr, config, gardenDirPath, plugins = [] } = opts + if (!config) { config = await findProjectConfig(currentDirectory) @@ -535,10 +542,10 @@ export class Garden { } const providers = await this.resolveProviders(false, [name]) - const provider = findByName(providers, name) + const provider = providers[name] if (!provider) { - const providerNames = providers.map((p) => p.name) + const providerNames = Object.keys(providers) throw new PluginError( `Could not find provider '${name}' in environment '${this.environmentName}' ` + `(configured providers: ${providerNames.join(", ")})`, @@ -552,7 +559,7 @@ export class Garden { return provider } - async resolveProviders(forceInit = false, names?: string[]): Promise { + async resolveProviders(forceInit = false, names?: string[]): Promise { let providers: Provider[] = [] await this.asyncLock.acquire("resolve-providers", async () => { @@ -650,7 +657,7 @@ export class Garden { this.log.silly(`Resolved providers: ${providers.map((p) => p.name).join(", ")}`) }) - return providers + return keyBy(providers, "name") } async getWorkflowConfig(name: string): Promise { @@ -673,7 +680,7 @@ export class Garden { */ async getEnvironmentStatus() { const providers = await this.resolveProviders() - return fromPairs(providers.map((p) => [p.name, p.status])) + return mapValues(providers, (p) => p.status) } async getActionRouter() { @@ -775,7 +782,7 @@ export class Garden { } // Walk through all plugins in dependency order, and allow them to augment the graph - for (const provider of getDependencyOrder(providers, this.registeredPlugins)) { + for (const provider of getDependencyOrder(Object.values(providers), this.registeredPlugins)) { // Skip the routine if the provider doesn't have the handler const handler = await actions.getActionHandler({ actionType: "augmentGraph", @@ -802,7 +809,7 @@ export class Garden { const configContext = new ModuleConfigContext({ garden: this, - resolvedProviders: providers, + resolvedProviders: keyBy(providers, "name"), variables: this.variables, secrets: this.secrets, dependencyConfigs: resolvedModules, @@ -1148,7 +1155,7 @@ export class Garden { return { environmentName: this.environmentName, namespace: this.namespace, - providers: await this.resolveProviders(), + providers: Object.values(await this.resolveProviders()).map((p) => omit(p, ["tools"])), variables: this.variables, moduleConfigs: sortBy( modules.map((m) => m._config), @@ -1166,7 +1173,7 @@ export class Garden { export interface ConfigDump { environmentName: string namespace?: string - providers: Provider[] + providers: Omit[] variables: DeepPrimitiveMap moduleConfigs: ModuleConfig[] workflowConfigs: WorkflowConfig[] diff --git a/garden-service/src/logger/writers/base.ts b/garden-service/src/logger/writers/base.ts index 3d785821a07..17d7f3045d1 100644 --- a/garden-service/src/logger/writers/base.ts +++ b/garden-service/src/logger/writers/base.ts @@ -13,7 +13,7 @@ import { LogLevel } from "../log-node" export abstract class Writer { abstract type: string - constructor(public level: LogLevel = LogLevel.info) {} + constructor(public level: LogLevel = LogLevel.info, public output = process.stdout) {} abstract onGraphChange(entry: LogEntry, logger: Logger): void abstract stop(): void diff --git a/garden-service/src/logger/writers/basic-terminal-writer.ts b/garden-service/src/logger/writers/basic-terminal-writer.ts index dda2713e8f1..4873b6e7cf0 100644 --- a/garden-service/src/logger/writers/basic-terminal-writer.ts +++ b/garden-service/src/logger/writers/basic-terminal-writer.ts @@ -21,7 +21,7 @@ export class BasicTerminalWriter extends Writer { onGraphChange(entry: LogEntry, logger: Logger) { const out = basicRender(entry, logger) if (out) { - process.stdout.write(out) + this.output.write(out) } } diff --git a/garden-service/src/outputs.ts b/garden-service/src/outputs.ts index dc8b9983176..f3f68693c9e 100644 --- a/garden-service/src/outputs.ts +++ b/garden-service/src/outputs.ts @@ -64,7 +64,7 @@ export async function resolveProjectOutputs(garden: Garden, log: LogEntry): Prom garden.rawOutputs, new OutputConfigContext({ garden, - resolvedProviders: [], + resolvedProviders: {}, variables: garden.variables, secrets: garden.secrets, modules: [], diff --git a/garden-service/src/plugins.ts b/garden-service/src/plugins.ts index b5424fe2c8f..ddceabd1e06 100644 --- a/garden-service/src/plugins.ts +++ b/garden-service/src/plugins.ts @@ -186,6 +186,16 @@ function resolvePlugin(plugin: GardenPlugin, loadedPlugins: PluginMap, configs: } } + // Add tools from base (unless they're overridden, in which case we ignore the one from the base) + resolved.tools = [...(plugin.tools || [])] + + for (const baseTool of base.tools || []) { + const tool = findByName(resolved.tools, baseTool.name) + if (!tool) { + resolved.tools.push(baseTool) + } + } + // If the base is not expressly configured for the environment, we pull and coalesce its module declarations. // We also make sure the plugin doesn't redeclare a module type from the base. resolved.createModuleTypes = [...plugin.createModuleTypes] diff --git a/garden-service/src/plugins/conftest/conftest.ts b/garden-service/src/plugins/conftest/conftest.ts index 8186e5e58d0..7d1919ba080 100644 --- a/garden-service/src/plugins/conftest/conftest.ts +++ b/garden-service/src/plugins/conftest/conftest.ts @@ -13,7 +13,6 @@ import { joi, joiIdentifier } from "../../config/common" import { dedent, naturalList } from "../../util/string" import { TestModuleParams } from "../../types/plugin/module/testModule" import { Module } from "../../types/module" -import { BinaryCmd } from "../../util/ext-tools" import chalk from "chalk" import { baseBuildSpecSchema } from "../../config/module" import { matchGlobs, listDirectory } from "../../util/fs" @@ -167,7 +166,7 @@ export const gardenPlugin = createGardenPlugin({ } args.push(...files) - const result = await conftest.exec({ log, args, ignoreError: true, cwd: buildPath }) + const result = await provider.tools.conftest.exec({ log, args, ignoreError: true, cwd: buildPath }) let success = true let parsed: any = [] @@ -246,34 +245,43 @@ export const gardenPlugin = createGardenPlugin({ }, }, ], -}) - -const conftest = new BinaryCmd({ - name: "conftest", - specs: { - darwin: { - url: "https://github.com/instrumenta/conftest/releases/download/v0.15.0/conftest_0.15.0_Darwin_x86_64.tar.gz", - sha256: "73cea42e467edf7bec58648514096f5975353b0523a5f2b309833ff4a972765e", - extract: { - format: "tar", - targetPath: ["conftest"], - }, - }, - linux: { - url: "https://github.com/instrumenta/conftest/releases/download/v0.15.0/conftest_0.15.0_Linux_x86_64.tar.gz", - sha256: "23c6af69dcd2c9fe935ee3cd5652cc14ffc9d7cf0fd55d4abc6a5c3bd470b692", - extract: { - format: "tar", - targetPath: ["conftest"], - }, - }, - win32: { - url: "https://github.com/instrumenta/conftest/releases/download/v0.15.0/conftest_0.15.0_Windows_x86_64.zip", - sha256: "c452bb4b71d6fbf5d918e1b3ed28092f7bc3a157f44e0ecd6fa1968e1cad4bec", - extract: { - format: "zip", - targetPath: ["conftest.exe"], - }, + tools: [ + { + name: "conftest", + description: "A rego-based configuration validator.", + type: "binary", + builds: [ + { + platform: "darwin", + architecture: "amd64", + url: "https://github.com/instrumenta/conftest/releases/download/v0.15.0/conftest_0.15.0_Darwin_x86_64.tar.gz", + sha256: "73cea42e467edf7bec58648514096f5975353b0523a5f2b309833ff4a972765e", + extract: { + format: "tar", + targetPath: "conftest", + }, + }, + { + platform: "linux", + architecture: "amd64", + url: "https://github.com/instrumenta/conftest/releases/download/v0.15.0/conftest_0.15.0_Linux_x86_64.tar.gz", + sha256: "23c6af69dcd2c9fe935ee3cd5652cc14ffc9d7cf0fd55d4abc6a5c3bd470b692", + extract: { + format: "tar", + targetPath: "conftest", + }, + }, + { + platform: "windows", + architecture: "amd64", + url: "https://github.com/instrumenta/conftest/releases/download/v0.15.0/conftest_0.15.0_Windows_x86_64.zip", + sha256: "c452bb4b71d6fbf5d918e1b3ed28092f7bc3a157f44e0ecd6fa1968e1cad4bec", + extract: { + format: "zip", + targetPath: "conftest.exe", + }, + }, + ], }, - }, + ], }) diff --git a/garden-service/src/plugins/container/build.ts b/garden-service/src/plugins/container/build.ts index 5522bc03351..aab1e5b2048 100644 --- a/garden-service/src/plugins/container/build.ts +++ b/garden-service/src/plugins/container/build.ts @@ -13,9 +13,11 @@ import { GetBuildStatusParams } from "../../types/plugin/module/getBuildStatus" import { BuildModuleParams } from "../../types/plugin/module/build" import { LogLevel } from "../../logger/log-node" import { createOutputStream } from "../../util/util" +import { ContainerProvider } from "./container" -export async function getContainerBuildStatus({ module, log }: GetBuildStatusParams) { - const identifier = await containerHelpers.imageExistsLocally(module, log) +export async function getContainerBuildStatus({ ctx, module, log }: GetBuildStatusParams) { + const containerProvider = ctx.provider as ContainerProvider + const identifier = await containerHelpers.imageExistsLocally(module, log, containerProvider) if (identifier) { log.debug({ @@ -28,19 +30,20 @@ export async function getContainerBuildStatus({ module, log }: GetBuildStatusPar return { ready: !!identifier } } -export async function buildContainerModule({ module, log }: BuildModuleParams) { +export async function buildContainerModule({ ctx, module, log }: BuildModuleParams) { containerHelpers.checkDockerServerVersion(await containerHelpers.getDockerVersion()) + const containerProvider = ctx.provider as ContainerProvider const buildPath = module.buildPath const image = module.spec.image const hasDockerfile = await containerHelpers.hasDockerfile(module) if (!!image && !hasDockerfile) { - if (await containerHelpers.imageExistsLocally(module, log)) { + if (await containerHelpers.imageExistsLocally(module, log, containerProvider)) { return { fresh: false } } log.setState(`Pulling image ${image}...`) - await containerHelpers.pullImage(module, log) + await containerHelpers.pullImage(module, log, containerProvider) return { fetched: true } } @@ -68,9 +71,13 @@ export async function buildContainerModule({ module, log }: BuildModuleParams export const containerModuleOutputsSchema = () => joi.object().keys({ @@ -242,4 +246,44 @@ export const gardenPlugin = createGardenPlugin({ }, }, ], + tools: [ + { + name: "docker", + description: "The official Docker CLI.", + type: "binary", + builds: [ + { + platform: "darwin", + architecture: "amd64", + url: "https://download.docker.com/mac/static/stable/x86_64/docker-19.03.6.tgz", + sha256: "82d279c6a2df05c2bb628607f4c3eacb5a7447be6d5f2a2f65643fbb6ed2f9af", + extract: { + format: "tar", + targetPath: "docker/docker", + }, + }, + { + platform: "linux", + architecture: "amd64", + url: "https://download.docker.com/linux/static/stable/x86_64/docker-19.03.6.tgz", + sha256: "34ff89ce917796594cd81149b1777d07786d297ffd0fef37a796b5897052f7cc", + extract: { + format: "tar", + targetPath: "docker/docker", + }, + }, + { + platform: "windows", + architecture: "amd64", + url: + "https://github.com/rgl/docker-ce-windows-binaries-vagrant/releases/download/v19.03.6/docker-19.03.6.zip", + sha256: "b4591baa2b7016af9ff3328a26146e4db3e6ce3fbe0503a7fd87363f29d63f5c", + extract: { + format: "zip", + targetPath: "docker/docker.exe", + }, + }, + ], + }, + ], }) diff --git a/garden-service/src/plugins/container/helpers.ts b/garden-service/src/plugins/container/helpers.ts index cc005d56ba8..8960dc4c8ef 100644 --- a/garden-service/src/plugins/container/helpers.ts +++ b/garden-service/src/plugins/container/helpers.ts @@ -11,7 +11,7 @@ import { readFile, pathExists, lstat } from "fs-extra" import semver from "semver" import { parse, CommandEntry } from "docker-file-parser" import isGlob from "is-glob" -import { ConfigurationError, RuntimeError, InternalError } from "../../exceptions" +import { ConfigurationError, RuntimeError } from "../../exceptions" import { splitFirst, spawn, splitLast, SpawnOutput } from "../../util/util" import { ModuleConfig } from "../../config/module" import { ContainerModule, ContainerRegistryConfig, defaultTag, defaultNamespace, ContainerModuleConfig } from "./config" @@ -21,9 +21,9 @@ import { flatten, uniq, fromPairs } from "lodash" import { LogEntry } from "../../logger/log-entry" import chalk from "chalk" import isUrl from "is-url" -import { BinaryCmd } from "../../util/ext-tools" import titleize from "titleize" import { stripQuotes } from "../../util/string" +import { ContainerProvider } from "./container" interface DockerVersion { client?: string @@ -208,14 +208,20 @@ const helpers = { } }, - async pullImage(module: ContainerModule, log: LogEntry) { + async pullImage(module: ContainerModule, log: LogEntry, provider: ContainerProvider) { const identifier = await helpers.getPublicImageId(module) - await helpers.dockerCli(module.buildPath, ["pull", identifier], log) + await helpers.dockerCli({ cwd: module.buildPath, args: ["pull", identifier], log, containerProvider: provider }) }, - async imageExistsLocally(module: ContainerModule, log: LogEntry) { + async imageExistsLocally(module: ContainerModule, log: LogEntry, provider: ContainerProvider) { const identifier = await helpers.getLocalImageId(module) - const exists = (await helpers.dockerCli(module.buildPath, ["images", identifier, "-q"], log)).stdout.length > 0 + const result = await helpers.dockerCli({ + cwd: module.buildPath, + args: ["images", identifier, "-q"], + log, + containerProvider: provider, + }) + const exists = result.stdout!.length > 0 return exists ? identifier : null }, @@ -246,21 +252,6 @@ const helpers = { return fromPairs(results) }, - /** - * Asserts that the specified docker client version meets the minimum requirements. - */ - checkDockerClientVersion(version: DockerVersion) { - if (!version.client) { - // This should not occur in normal usage, so it is classed as an internal error - throw new InternalError(`Docker client is not installed.`, version) - } else if (!checkMinDockerVersion(version.client, minDockerVersion.client!)) { - throw new RuntimeError( - `Docker client needs to be version ${minDockerVersion.client} or newer (got ${version.client})`, - version - ) - } - }, - /** * Asserts that the specified docker client version meets the minimum requirements. */ @@ -277,38 +268,34 @@ const helpers = { } }, - async getDockerCliPath(log: LogEntry) { - // Check if docker is already installed - try { - const version = await helpers.getDockerVersion("docker") - helpers.checkDockerClientVersion(version) - return "docker" - } catch (_) { - // Need to fetch a docker client - return dockerBin.getPath(log) - } - }, - - async dockerCli( - cwd: string, - args: string[], - log: LogEntry, - { - ignoreError = false, - outputStream, - timeout = DEFAULT_BUILD_TIMEOUT, - }: { ignoreError?: boolean; outputStream?: Writable; timeout?: number } = {} - ) { - // Check if docker is already installed - const cliPath = await helpers.getDockerCliPath(log) + async dockerCli({ + cwd, + args, + log, + containerProvider, + ignoreError = false, + outputStream, + timeout = DEFAULT_BUILD_TIMEOUT, + }: { + cwd: string + args: string[] + log: LogEntry + containerProvider: ContainerProvider + ignoreError?: boolean + outputStream?: Writable + timeout?: number + }) { + const docker = containerProvider.tools.docker try { - const res = await spawn(cliPath, args, { + const res = await docker.spawnAndWait({ + args, cwd, + env: { ...process.env, DOCKER_CLI_EXPERIMENTAL: "enabled" }, ignoreError, + log, stdout: outputStream, timeout, - env: { ...process.env, DOCKER_CLI_EXPERIMENTAL: "enabled" }, }) return res } catch (err) { @@ -436,33 +423,3 @@ function fixDockerVersionString(v: string) { function getDockerfilePath(basePath: string, dockerfile = "Dockerfile") { return join(basePath, dockerfile) } - -export const dockerBin = new BinaryCmd({ - name: "docker", - specs: { - darwin: { - url: "https://download.docker.com/mac/static/stable/x86_64/docker-19.03.6.tgz", - sha256: "82d279c6a2df05c2bb628607f4c3eacb5a7447be6d5f2a2f65643fbb6ed2f9af", - extract: { - format: "tar", - targetPath: ["docker", "docker"], - }, - }, - linux: { - url: "https://download.docker.com/linux/static/stable/x86_64/docker-19.03.6.tgz", - sha256: "34ff89ce917796594cd81149b1777d07786d297ffd0fef37a796b5897052f7cc", - extract: { - format: "tar", - targetPath: ["docker", "docker"], - }, - }, - win32: { - url: "https://github.com/rgl/docker-ce-windows-binaries-vagrant/releases/download/v19.03.6/docker-19.03.6.zip", - sha256: "b4591baa2b7016af9ff3328a26146e4db3e6ce3fbe0503a7fd87363f29d63f5c", - extract: { - format: "zip", - targetPath: ["docker", "docker.exe"], - }, - }, - }, -}) diff --git a/garden-service/src/plugins/container/publish.ts b/garden-service/src/plugins/container/publish.ts index 142f4e7a208..0426d737f3b 100644 --- a/garden-service/src/plugins/container/publish.ts +++ b/garden-service/src/plugins/container/publish.ts @@ -9,12 +9,14 @@ import { ContainerModule } from "./config" import { PublishModuleParams } from "../../types/plugin/module/publishModule" import { containerHelpers } from "./helpers" +import { ContainerProvider } from "./container" -export async function publishContainerModule({ module, log }: PublishModuleParams) { +export async function publishContainerModule({ ctx, module, log }: PublishModuleParams) { if (!(await containerHelpers.hasDockerfile(module))) { log.setState({ msg: `Nothing to publish` }) return { published: false } } + const containerProvider = ctx.provider as ContainerProvider const localId = await containerHelpers.getLocalImageId(module) const remoteId = await containerHelpers.getPublicImageId(module) @@ -22,11 +24,16 @@ export async function publishContainerModule({ module, log }: PublishModuleParam log.setState({ msg: `Publishing image ${remoteId}...` }) if (localId !== remoteId) { - await containerHelpers.dockerCli(module.buildPath, ["tag", localId, remoteId], log) + await containerHelpers.dockerCli({ + cwd: module.buildPath, + args: ["tag", localId, remoteId], + log, + containerProvider, + }) } // TODO: stream output to log if at debug log level - await containerHelpers.dockerCli(module.buildPath, ["push", remoteId], log) + await containerHelpers.dockerCli({ cwd: module.buildPath, args: ["push", remoteId], log, containerProvider }) return { published: true, message: `Published ${remoteId}` } } diff --git a/garden-service/src/plugins/hadolint/hadolint.ts b/garden-service/src/plugins/hadolint/hadolint.ts index 358102ec308..c39849a92a2 100644 --- a/garden-service/src/plugins/hadolint/hadolint.ts +++ b/garden-service/src/plugins/hadolint/hadolint.ts @@ -15,7 +15,6 @@ import { joi } from "../../config/common" import { dedent, splitLines, naturalList } from "../../util/string" import { TestModuleParams } from "../../types/plugin/module/testModule" import { Module } from "../../types/module" -import { BinaryCmd } from "../../util/ext-tools" import { STATIC_DIR } from "../../constants" import { padStart, padEnd } from "lodash" import chalk from "chalk" @@ -188,7 +187,7 @@ export const gardenPlugin = createGardenPlugin({ } const args = ["--config", configPath, "--format", "json", dockerfilePath] - const result = await hadolint.exec({ log, args, ignoreError: true }) + const result = await ctx.provider.tools.hadolint.exec({ log, args, ignoreError: true }) let success = true @@ -255,22 +254,31 @@ export const gardenPlugin = createGardenPlugin({ }, }, ], -}) - -const hadolint = new BinaryCmd({ - name: "hadolint", - specs: { - darwin: { - url: "https://github.com/hadolint/hadolint/releases/download/v1.17.2/hadolint-Darwin-x86_64", - sha256: "da3bd1fae47f1ba4c4bca6a86d2c70bdbd6705308bd300d1f897c162bc32189a", - }, - linux: { - url: "https://github.com/hadolint/hadolint/releases/download/v1.17.2/hadolint-Linux-x86_64", - sha256: "b23e4d0e8964774cc0f4dd7ff81f1d05b5d7538b0b80dae5235b1239ab60749d", - }, - win32: { - url: "https://github.com/hadolint/hadolint/releases/download/v1.17.2/hadolint-Windows-x86_64.exe", - sha256: "8ba81d1fe79b91afb7ee16ac4e9fc6635646c2f770071d1ba924a8d26debe298", + tools: [ + { + name: "hadolint", + description: "A Dockerfile linter.", + type: "binary", + builds: [ + { + platform: "darwin", + architecture: "amd64", + url: "https://github.com/hadolint/hadolint/releases/download/v1.17.2/hadolint-Darwin-x86_64", + sha256: "da3bd1fae47f1ba4c4bca6a86d2c70bdbd6705308bd300d1f897c162bc32189a", + }, + { + platform: "linux", + architecture: "amd64", + url: "https://github.com/hadolint/hadolint/releases/download/v1.17.2/hadolint-Linux-x86_64", + sha256: "b23e4d0e8964774cc0f4dd7ff81f1d05b5d7538b0b80dae5235b1239ab60749d", + }, + { + platform: "windows", + architecture: "amd64", + url: "https://github.com/hadolint/hadolint/releases/download/v1.17.2/hadolint-Windows-x86_64.exe", + sha256: "8ba81d1fe79b91afb7ee16ac4e9fc6635646c2f770071d1ba924a8d26debe298", + }, + ], }, - }, + ], }) diff --git a/garden-service/src/plugins/kubernetes/api.ts b/garden-service/src/plugins/kubernetes/api.ts index d944fafc14b..5237f70d7ac 100644 --- a/garden-service/src/plugins/kubernetes/api.ts +++ b/garden-service/src/plugins/kubernetes/api.ts @@ -482,7 +482,7 @@ export async function getKubeConfig(log: LogEntry, provider: KubernetesProvider) kubeConfigStr = (await readFile(provider.config.kubeconfig)).toString() } else { // We use kubectl for this, to support merging multiple paths in the KUBECONFIG env var - kubeConfigStr = await kubectl.stdout({ log, provider, args: ["config", "view", "--raw"] }) + kubeConfigStr = await kubectl(provider).stdout({ log, args: ["config", "view", "--raw"] }) } return safeLoad(kubeConfigStr) } catch (error) { diff --git a/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts b/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts index f5182b5226e..f69dd35f7a6 100644 --- a/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts +++ b/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts @@ -455,9 +455,8 @@ async function execInBuildSync({ provider, log, args, timeout, podName }: Builde log.verbose(`Running: kubectl ${execCmd.join(" ")}`) - return kubectl.exec({ + return kubectl(provider).exec({ args: execCmd, - provider, log, namespace: systemNamespace, timeout, diff --git a/garden-service/src/plugins/kubernetes/commands/pull-image.ts b/garden-service/src/plugins/kubernetes/commands/pull-image.ts index a7cb698b3f2..ba8a3d3a85f 100644 --- a/garden-service/src/plugins/kubernetes/commands/pull-image.ts +++ b/garden-service/src/plugins/kubernetes/commands/pull-image.ts @@ -24,6 +24,7 @@ import { inClusterRegistryHostname } from "../constants" import { getAppNamespace, getSystemNamespace } from "../namespace" import { makePodName, skopeoImage, getSkopeoContainer, getDockerAuthVolume } from "../util" import { getRegistryPortForward } from "../container/util" +import { ContainerProvider } from "../../container/container" export const pullImage: PluginCommand = { name: "pull-image", @@ -120,10 +121,16 @@ async function pullFromInClusterRegistry( // https://github.com/docker/for-mac/issues/3611 host: `local.app.garden:${fwd.localPort}`, }) + const containerProvider = k8sCtx.provider.dependencies.container as ContainerProvider - await containerHelpers.dockerCli(module.buildPath, ["pull", pullImageId], log) - await containerHelpers.dockerCli(module.buildPath, ["tag", pullImageId, localId], log) - await containerHelpers.dockerCli(module.buildPath, ["rmi", pullImageId], log) + await containerHelpers.dockerCli({ cwd: module.buildPath, args: ["pull", pullImageId], log, containerProvider }) + await containerHelpers.dockerCli({ + cwd: module.buildPath, + args: ["tag", pullImageId, localId], + log, + containerProvider, + }) + await containerHelpers.dockerCli({ cwd: module.buildPath, args: ["rmi", pullImageId], log, containerProvider }) } async function pullFromExternalRegistry(ctx: KubernetesPluginContext, module: Module, log: LogEntry, localId: string) { @@ -133,6 +140,7 @@ async function pullFromExternalRegistry(ctx: KubernetesPluginContext, module: Mo const systemNamespace = await getSystemNamespace(ctx.provider, log) const imageId = await containerHelpers.getDeploymentImageId(module, ctx.provider.config.deploymentRegistry) const tarName = `${module.name}-${module.version.versionString}` + const containerProvider = ctx.provider.dependencies.container as ContainerProvider const skopeoCommand = [ "skopeo", @@ -147,9 +155,9 @@ async function pullFromExternalRegistry(ctx: KubernetesPluginContext, module: Mo try { await pullImageFromRegistry(runner, skopeoCommand.join(" "), log) - await importImage(module, runner, tarName, imageId, log) - await containerHelpers.dockerCli(module.buildPath, ["tag", imageId, localId], log) - await containerHelpers.dockerCli(module.buildPath, ["rmi", imageId], log) + await importImage({ module, runner, tarName, imageId, log, containerProvider }) + await containerHelpers.dockerCli({ cwd: module.buildPath, args: ["tag", imageId, localId], log, containerProvider }) + await containerHelpers.dockerCli({ cwd: module.buildPath, args: ["rmi", imageId], log, containerProvider }) } catch (err) { throw new RuntimeError(`Failed pulling image for module ${module.name} with image id ${imageId}: ${err}`, { err, @@ -160,7 +168,21 @@ async function pullFromExternalRegistry(ctx: KubernetesPluginContext, module: Mo } } -async function importImage(module: Module, runner: PodRunner, tarName: string, imageId: string, log: LogEntry) { +async function importImage({ + module, + runner, + tarName, + imageId, + log, + containerProvider, +}: { + module: Module + runner: PodRunner + tarName: string + imageId: string + log: LogEntry + containerProvider: ContainerProvider +}) { const sourcePath = `/${tarName}` const getOutputCommand = ["cat", sourcePath] await tmp.withFile(async ({ path }) => { @@ -175,7 +197,7 @@ async function importImage(module: Module, runner: PodRunner, tarName: string, i }) const args = ["import", path, imageId] - await containerHelpers.dockerCli(module.buildPath, args, log) + await containerHelpers.dockerCli({ cwd: module.buildPath, args, log, containerProvider }) }) } diff --git a/garden-service/src/plugins/kubernetes/container/build.ts b/garden-service/src/plugins/kubernetes/container/build.ts index ca234784bd8..257c2e8df25 100644 --- a/garden-service/src/plugins/kubernetes/container/build.ts +++ b/garden-service/src/plugins/kubernetes/container/build.ts @@ -43,6 +43,7 @@ import { dedent } from "../../../util/string" import chalk = require("chalk") import { loadImageToMicrok8s, getMicrok8sImageStatus } from "../local/microk8s" import { RunResult } from "../../../types/plugin/base" +import { ContainerProvider } from "../../container/container" const kanikoImage = "gcr.io/kaniko-project/executor:debug-v0.22.0" @@ -78,11 +79,18 @@ const buildStatusHandlers: { [mode in ContainerBuildMode]: BuildStatusHandler } "local-docker": async (params) => { const { ctx, module, log } = params const k8sCtx = ctx as KubernetesPluginContext + const containerProvider = k8sCtx.provider.dependencies.container as ContainerProvider const deploymentRegistry = k8sCtx.provider.config.deploymentRegistry if (deploymentRegistry) { const args = await getManifestInspectArgs(module, deploymentRegistry) - const res = await containerHelpers.dockerCli(module.buildPath, args, log, { ignoreError: true }) + const res = await containerHelpers.dockerCli({ + cwd: module.buildPath, + args, + log, + containerProvider, + ignoreError: true, + }) // Non-zero exit code can both mean the manifest is not found, and any other unexpected error if (res.code !== 0 && !res.all.includes("no such manifest")) { @@ -95,7 +103,7 @@ const buildStatusHandlers: { [mode in ContainerBuildMode]: BuildStatusHandler } const localId = await containerHelpers.getLocalImageId(module) return getMicrok8sImageStatus(localId) } else { - return getContainerBuildStatus(params) + return getContainerBuildStatus({ ...params, ctx: { ...ctx, provider: ctx.provider.dependencies.container } }) } }, @@ -185,14 +193,15 @@ type BuildHandler = (params: BuildModuleParams) => Promise { const { ctx, module, log } = params const provider = ctx.provider as KubernetesProvider - const buildResult = await buildContainerModule(params) + const containerProvider = provider.dependencies.container as ContainerProvider + const buildResult = await buildContainerModule({ ...params, ctx: { ...ctx, provider: containerProvider } }) if (!provider.config.deploymentRegistry) { if (provider.config.clusterType === "kind") { await loadImageToKind(buildResult, provider.config) } else if (provider.config.clusterType === "microk8s") { const imageId = await containerHelpers.getLocalImageId(module) - await loadImageToMicrok8s({ module, imageId, log }) + await loadImageToMicrok8s({ module, imageId, log, containerProvider }) } return buildResult } @@ -206,8 +215,8 @@ const localBuild: BuildHandler = async (params) => { log.setState({ msg: `Pushing image ${remoteId} to cluster...` }) - await containerHelpers.dockerCli(module.buildPath, ["tag", localId, remoteId], log) - await containerHelpers.dockerCli(module.buildPath, ["push", remoteId], log) + await containerHelpers.dockerCli({ cwd: module.buildPath, args: ["tag", localId, remoteId], log, containerProvider }) + await containerHelpers.dockerCli({ cwd: module.buildPath, args: ["push", remoteId], log, containerProvider }) return buildResult } @@ -408,10 +417,9 @@ export async function execInPod({ log.verbose(`Running: kubectl ${execCmd.join(" ")}`) - return kubectl.exec({ + return kubectl(provider).exec({ args: execCmd, ignoreError, - provider, log, namespace: systemNamespace, timeout, diff --git a/garden-service/src/plugins/kubernetes/container/exec.ts b/garden-service/src/plugins/kubernetes/container/exec.ts index f77525e643b..7e5da68ed74 100644 --- a/garden-service/src/plugins/kubernetes/container/exec.ts +++ b/garden-service/src/plugins/kubernetes/container/exec.ts @@ -80,9 +80,8 @@ export async function execInWorkload({ } const kubecmd = ["exec", ...opts, pod.metadata.name, "--", ...command] - const res = await kubectl.spawnAndWait({ + const res = await kubectl(provider).spawnAndWait({ log, - provider, namespace, args: kubecmd, ignoreError: true, diff --git a/garden-service/src/plugins/kubernetes/container/publish.ts b/garden-service/src/plugins/kubernetes/container/publish.ts index f8bd2ec4d6b..5456c7038ae 100644 --- a/garden-service/src/plugins/kubernetes/container/publish.ts +++ b/garden-service/src/plugins/kubernetes/container/publish.ts @@ -12,11 +12,13 @@ import { containerHelpers } from "../../container/helpers" import { KubernetesPluginContext } from "../config" import { publishContainerModule } from "../../container/publish" import { getRegistryPortForward } from "./util" +import { ContainerProvider } from "../../container/container" export async function k8sPublishContainerModule(params: PublishModuleParams) { const { ctx, module, log } = params const k8sCtx = ctx as KubernetesPluginContext const provider = k8sCtx.provider + const containerProvider = provider.dependencies.container as ContainerProvider if (!(await containerHelpers.hasDockerfile(module))) { log.setState({ msg: `Nothing to publish` }) @@ -40,12 +42,22 @@ export async function k8sPublishContainerModule(params: PublishModuleParams = await kubectl.json({ + const res: KubernetesList = await kubectl(provider).json({ args: ["get", manifest.kind, "-l", selector], log, namespace, - provider, }) const list = res.items.filter((r) => r.metadata.annotations![gardenAnnotationKey("hot-reload")] === "true") diff --git a/garden-service/src/plugins/kubernetes/integrations/cert-manager.ts b/garden-service/src/plugins/kubernetes/integrations/cert-manager.ts index 727dcb16480..535e626ce9f 100644 --- a/garden-service/src/plugins/kubernetes/integrations/cert-manager.ts +++ b/garden-service/src/plugins/kubernetes/integrations/cert-manager.ts @@ -144,7 +144,7 @@ export function isCertificateReady(cert) { */ export async function getAllCertificates(log: LogEntry, provider: KubernetesProvider, namespace: string) { const args = ["get", "certificates", "--namespace", namespace] - return kubectl.json({ log, provider, args }) + return kubectl(provider).json({ log, args }) } /** diff --git a/garden-service/src/plugins/kubernetes/kubectl.ts b/garden-service/src/plugins/kubernetes/kubectl.ts index 04e7dc7c43f..324008be896 100644 --- a/garden-service/src/plugins/kubernetes/kubectl.ts +++ b/garden-service/src/plugins/kubernetes/kubectl.ts @@ -8,12 +8,13 @@ import _spawn from "cross-spawn" import { encodeYamlMulti } from "../../util/util" -import { BinaryCmd, ExecParams } from "../../util/ext-tools" +import { ExecParams, PluginTool } from "../../util/ext-tools" import { LogEntry } from "../../logger/log-entry" import { KubernetesProvider } from "./config" import { KubernetesResource } from "./types" import { gardenAnnotationKey } from "../../util/string" import { hashManifest } from "./util" +import { PluginToolSpec } from "../../types/plugin/tools" export interface ApplyParams { log: LogEntry @@ -57,7 +58,7 @@ export async function apply({ args.push("--output=json", "-f", "-") !validate && args.push("--validate=false") - const result = await kubectl.stdout({ log, provider, namespace, args, input }) + const result = await kubectl(provider).stdout({ log, namespace, args, input }) try { return JSON.parse(result) @@ -88,7 +89,7 @@ export async function deleteResources({ includeUninitialized && args.push("--include-uninitialized") - return kubectl.stdout({ provider, namespace, args, log }) + return kubectl(provider).stdout({ namespace, args, log }) } export async function deleteObjectsBySelector({ @@ -110,12 +111,11 @@ export async function deleteObjectsBySelector({ includeUninitialized && args.push("--include-uninitialized") - return kubectl.stdout({ provider, namespace, args, log }) + return kubectl(provider).stdout({ namespace, args, log }) } interface KubectlParams extends ExecParams { log: LogEntry - provider: KubernetesProvider namespace?: string configPath?: string args: string[] @@ -126,10 +126,13 @@ interface KubectlSpawnParams extends KubectlParams { wait?: boolean } -class Kubectl extends BinaryCmd { - async exec(params: KubectlParams) { - this.prepareArgs(params) - return super.exec(params) +export function kubectl(provider: KubernetesProvider) { + return new Kubectl(provider.tools.kubectl.spec, provider) +} + +class Kubectl extends PluginTool { + constructor(spec: PluginToolSpec, private provider: KubernetesProvider) { + super(spec) } async stdout(params: KubectlParams) { @@ -137,6 +140,11 @@ class Kubectl extends BinaryCmd { return super.stdout(params) } + async exec(params: KubectlParams) { + this.prepareArgs(params) + return super.exec(params) + } + async spawn(params: KubectlParams) { this.prepareArgs(params) return super.spawn(params) @@ -158,12 +166,12 @@ class Kubectl extends BinaryCmd { } private prepareArgs(params: KubectlParams) { - const { provider, namespace, configPath, args } = params + const { namespace, configPath, args } = params - const opts: string[] = [`--context=${provider.config.context}`] + const opts: string[] = [`--context=${this.provider.config.context}`] - if (provider.config.kubeconfig) { - opts.push(`--kubeconfig=${provider.config.kubeconfig}`) + if (this.provider.config.kubeconfig) { + opts.push(`--kubeconfig=${this.provider.config.kubeconfig}`) } if (namespace) { @@ -178,21 +186,28 @@ class Kubectl extends BinaryCmd { } } -export const kubectl = new Kubectl({ +export const kubectlSpec: PluginToolSpec = { name: "kubectl", - defaultTimeout: KUBECTL_DEFAULT_TIMEOUT, - specs: { - darwin: { + description: "The official Kubernetes CLI.", + type: "binary", + builds: [ + { + platform: "darwin", + architecture: "amd64", url: "https://storage.googleapis.com/kubernetes-release/release/v1.16.0/bin/darwin/amd64/kubectl", sha256: "a81b23abe67e70f8395ff7a3659bea6610fba98cda1126ef19e0a995f0075d54", }, - linux: { + { + platform: "linux", + architecture: "amd64", url: "https://storage.googleapis.com/kubernetes-release/release/v1.16.0/bin/linux/amd64/kubectl", sha256: "4fc8a7024ef17b907820890f11ba7e59a6a578fa91ea593ce8e58b3260f7fb88", }, - win32: { + { + platform: "windows", + architecture: "amd64", url: "https://storage.googleapis.com/kubernetes-release/release/v1.16.0/bin/windows/amd64/kubectl.exe", sha256: "a7e4e527735f5bc49ad80b92f4a9d3bb6aebd129f9a708baac80465ebc33a9bc", }, - }, -}) + ], +} diff --git a/garden-service/src/plugins/kubernetes/kubernetes.ts b/garden-service/src/plugins/kubernetes/kubernetes.ts index 31f2fea583a..c128e7b4f53 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes.ts @@ -17,7 +17,7 @@ import { containerHandlers } from "./container/handlers" import { kubernetesHandlers } from "./kubernetes-module/handlers" import { ConfigureProviderParams } from "../../types/plugin/provider/configureProvider" import { DebugInfo, GetDebugInfoParams } from "../../types/plugin/provider/getDebugInfo" -import { kubectl } from "./kubectl" +import { kubectl, kubectlSpec } from "./kubectl" import { KubernetesConfig, KubernetesPluginContext } from "./config" import { configSchema } from "./config" import { ConfigurationError } from "../../exceptions" @@ -39,6 +39,8 @@ import { DOCS_BASE_URL } from "../../constants" import { inClusterRegistryHostname } from "./constants" import { pvcModuleDefinition } from "./volumes/persistentvolumeclaim" import { getModuleTypeUrl, getProviderUrl } from "../../docs/common" +import { helm2Spec, helm3Spec, helm2to3Spec } from "./helm/helm-cli" +import { sternSpec } from "./logs" export async function configureProvider({ namespace, @@ -149,7 +151,7 @@ export async function debugInfo({ ctx, log, includeProject }: GetDebugInfoParams } const namespaces = await Bluebird.map(namespacesList, async (ns) => { const nsEntry = entry.info({ section: ns, msg: "collecting namespace configuration", status: "active" }) - const out = await kubectl.stdout({ log, provider, args: ["get", "all", "--namespace", ns, "--output", "json"] }) + const out = await kubectl(provider).stdout({ log, args: ["get", "all", "--namespace", ns, "--output", "json"] }) nsEntry.setSuccess({ msg: chalk.green(`Done (took ${log.getDuration(1)} sec)`), append: true }) return { namespace: ns, @@ -158,7 +160,7 @@ export async function debugInfo({ ctx, log, includeProject }: GetDebugInfoParams }) entry.setSuccess({ msg: chalk.green(`Done (took ${log.getDuration(1)} sec)`), append: true }) - const version = await kubectl.stdout({ log, provider, args: ["version", "--output", "json"] }) + const version = await kubectl(provider).stdout({ log, args: ["version", "--output", "json"] }) return { info: { version: JSON.parse(version), namespaces }, @@ -240,4 +242,5 @@ export const gardenPlugin = createGardenPlugin({ handlers: containerHandlers, }, ], + tools: [kubectlSpec, helm2Spec, helm3Spec, helm2to3Spec, sternSpec], }) diff --git a/garden-service/src/plugins/kubernetes/local/config.ts b/garden-service/src/plugins/kubernetes/local/config.ts index d5e617b4af2..d9fd3a67a82 100644 --- a/garden-service/src/plugins/kubernetes/local/config.ts +++ b/garden-service/src/plugins/kubernetes/local/config.ts @@ -47,7 +47,7 @@ export const configSchema = kubernetesConfigBase .description("The provider configuration for the local-kubernetes plugin.") export async function configureProvider(params: ConfigureProviderParams) { - const { base, log, projectName } = params + const { base, log, projectName, tools } = params let { config } = await base!(params) const _systemServices = config._systemServices @@ -55,10 +55,11 @@ export async function configureProvider(params: ConfigureProviderParams { try { // See https://microk8s.io/docs/registry-images for reference await tmp.withFile(async (file) => { - await containerHelpers.dockerCli(module.buildPath, ["save", "-o", file.path, imageId], log) + await containerHelpers.dockerCli({ + cwd: module.buildPath, + args: ["save", "-o", file.path, imageId], + log, + containerProvider, + }) await exec("microk8s.ctr", ["image", "import", file.path]) }) } catch (err) { diff --git a/garden-service/src/plugins/kubernetes/logs.ts b/garden-service/src/plugins/kubernetes/logs.ts index 1cbd79334fe..66e77cbaec8 100644 --- a/garden-service/src/plugins/kubernetes/logs.ts +++ b/garden-service/src/plugins/kubernetes/logs.ts @@ -19,10 +19,10 @@ import Stream from "ts-stream" import { LogEntry } from "../../logger/log-entry" import Bluebird from "bluebird" import { KubernetesProvider } from "./config" -import { BinaryCmd } from "../../util/ext-tools" import { kubectl } from "./kubectl" import { splitFirst } from "../../util/util" import { ChildProcess } from "child_process" +import { PluginToolSpec } from "../../types/plugin/tools" interface GetLogsBaseParams { defaultNamespace: string @@ -46,26 +46,31 @@ interface GetLogsParams extends GetLogsBaseParams { pod: KubernetesPod } -const STERN_NAME = "stern" -const STERN_TIME_OUT = 300 -const stern = new BinaryCmd({ - name: STERN_NAME, - defaultTimeout: STERN_TIME_OUT, - specs: { - darwin: { +export const sternSpec: PluginToolSpec = { + name: "stern", + description: "Utility CLI for streaming logs from Kubernetes.", + type: "binary", + builds: [ + { + platform: "darwin", + architecture: "amd64", url: "https://github.com/wercker/stern/releases/download/1.11.0/stern_darwin_amd64", sha256: "7aea3b6691d47b3fb844dfc402905790665747c1e6c02c5cabdd41994533d7e9", }, - linux: { + { + platform: "linux", + architecture: "amd64", url: "https://github.com/wercker/stern/releases/download/1.11.0/stern_linux_amd64", sha256: "e0b39dc26f3a0c7596b2408e4fb8da533352b76aaffdc18c7ad28c833c9eb7db", }, - win32: { + { + platform: "windows", + architecture: "amd64", url: "https://github.com/wercker/stern/releases/download/1.11.0/stern_windows_amd64.exe", sha256: "75708b9acf6ef0eeffbe1f189402adc0405f1402e6b764f1f5152ca288e3109e", }, - }, -}) + ], +} /** * Stream all logs for the given pod names and service. @@ -115,10 +120,9 @@ async function readLogs( kubectlArgs.push(`pod/${pod.metadata.name}`) - const proc = await kubectl.spawn({ + const proc = await kubectl(provider).spawn({ args: kubectlArgs, log, - provider, namespace: pod.metadata.namespace, }) @@ -157,7 +161,7 @@ async function followLogs( sternArgs.push(`${service.name}`) } - const proc = await stern.spawn({ + const proc = await provider.tools.stern.spawn({ args: sternArgs, log, }) diff --git a/garden-service/src/plugins/kubernetes/namespace.ts b/garden-service/src/plugins/kubernetes/namespace.ts index 9afbba6d634..852bb0f5a53 100644 --- a/garden-service/src/plugins/kubernetes/namespace.ts +++ b/garden-service/src/plugins/kubernetes/namespace.ts @@ -137,7 +137,7 @@ export async function prepareNamespaces({ ctx, log }: GetEnvironmentStatusParams try { // TODO: use API instead of kubectl (I just couldn't find which API call to make) - await kubectl.exec({ log, provider: k8sCtx.provider, args: ["version"] }) + await kubectl(k8sCtx.provider).exec({ log, args: ["version"] }) } catch (err) { log.setError("Error") diff --git a/garden-service/src/plugins/kubernetes/port-forward.ts b/garden-service/src/plugins/kubernetes/port-forward.ts index 3befaff504e..4f40d50f083 100644 --- a/garden-service/src/plugins/kubernetes/port-forward.ts +++ b/garden-service/src/plugins/kubernetes/port-forward.ts @@ -108,7 +108,7 @@ export async function getPortForward({ const portForwardArgs = ["port-forward", targetResource, portMapping] log.silly(`Running 'kubectl ${portForwardArgs.join(" ")}'`) - const proc = await kubectl.spawn({ log, provider: k8sCtx.provider, namespace, args: portForwardArgs }) + const proc = await kubectl(k8sCtx.provider).spawn({ log, namespace, args: portForwardArgs }) let output = "" return new Promise((resolve, reject) => { diff --git a/garden-service/src/plugins/kubernetes/run.ts b/garden-service/src/plugins/kubernetes/run.ts index 6a3eaf9cbd2..7d414a1c1b2 100644 --- a/garden-service/src/plugins/kubernetes/run.ts +++ b/garden-service/src/plugins/kubernetes/run.ts @@ -446,9 +446,8 @@ export class PodRunner extends PodRunnerParams { const startedAt = new Date() // TODO: use API library - const res = await kubectl.spawnAndWait({ + const res = await kubectl(this.provider).spawnAndWait({ log, - provider: this.provider, namespace: this.namespace, ignoreError, args: kubecmd, @@ -482,9 +481,8 @@ export class PodRunner extends PodRunnerParams { log.verbose(`Starting Pod ${this.podName} with command '${command.join(" ")}'`) // TODO: use API directly - this.proc = await kubectl.spawn({ + this.proc = await kubectl(this.provider).spawn({ log, - provider: this.provider, namespace: this.namespace, args: kubecmd, stdout, @@ -555,13 +553,12 @@ export class PodRunner extends PodRunnerParams { const startedAt = new Date() - const proc = await kubectl.spawn({ + const proc = await kubectl(this.provider).spawn({ args, namespace: this.namespace, ignoreError, input, log, - provider: this.provider, stdout, stderr, timeout, @@ -619,13 +616,12 @@ export class PodRunner extends PodRunnerParams { const startedAt = new Date() - const res = await kubectl.exec({ + const res = await kubectl(this.provider).exec({ args, namespace: this.namespace, ignoreError, input, log, - provider: this.provider, stdout, stderr, timeout, diff --git a/garden-service/src/plugins/maven-container/maven-container.ts b/garden-service/src/plugins/maven-container/maven-container.ts index 3f75d5fa64f..e38f5957060 100644 --- a/garden-service/src/plugins/maven-container/maven-container.ts +++ b/garden-service/src/plugins/maven-container/maven-container.ts @@ -25,8 +25,6 @@ import { STATIC_DIR } from "../../constants" import { xml2json } from "xml-js" import { containerModuleSpecSchema } from "../container/config" import { providerConfigBaseSchema } from "../../config/provider" -import { openJdks } from "./openjdk" -import { maven } from "./maven" import { LogEntry } from "../../logger/log-entry" import { dedent } from "../../util/string" import { ModuleConfig } from "../../config/module" @@ -78,7 +76,7 @@ const mavenKeys = { jdkVersion: joi .number() .integer() - .allow(...Object.keys(openJdks)) + .allow(8, 11, 13) .default(8) .description("The JDK version to use."), mvnOpts: joiArray(joi.string()).description("Options to add to the `mvn package` command when building."), @@ -181,7 +179,7 @@ async function getBuildStatus(params: GetBuildStatusParams async function build(params: BuildModuleParams) { // Run the maven build - const { base, module, log } = params + const { ctx, base, module, log } = params let { jarPath, jdkVersion, mvnOpts, useDefaultDockerfile, image } = module.spec // Fall back to using the image field @@ -206,7 +204,7 @@ async function build(params: BuildModuleParams) { log.setState(`Creating jar artifact...`) - const openJdk = openJdks[jdkVersion] + const openJdk = ctx.provider.tools["openjdk-" + jdkVersion] const openJdkPath = await openJdk.getPath(log) const mvnArgs = ["package", "--batch-mode", "--projects", ":" + artifactId, "--also-make", ...mvnOpts] @@ -215,7 +213,7 @@ async function build(params: BuildModuleParams) { // Maven has issues when running concurrent processes, so we're working around that with a lock. // TODO: http://takari.io/book/30-team-maven.html would be a more robust solution. await buildLock.acquire("mvn", async () => { - await maven.exec({ + await ctx.provider.tools.maven.exec({ args: mvnArgs, cwd: module.path, log, diff --git a/garden-service/src/plugins/maven-container/maven.ts b/garden-service/src/plugins/maven-container/maven.ts index c85eaa5abb5..82be910419a 100644 --- a/garden-service/src/plugins/maven-container/maven.ts +++ b/garden-service/src/plugins/maven-container/maven.ts @@ -6,22 +6,36 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { BinaryCmd, LibraryPlatformSpec } from "../../util/ext-tools" +import { PluginToolSpec } from "../../types/plugin/tools" -const spec: LibraryPlatformSpec = { +const spec = { url: "http://mirror.23media.de/apache/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz", sha256: "6a1b346af36a1f1a491c1c1a141667c5de69b42e6611d3687df26868bc0f4637", extract: { format: "tar", - targetPath: ["apache-maven-3.6.0", "bin", "mvn"], + targetPath: "apache-maven-3.6.0/bin/mvn", }, } -export const maven = new BinaryCmd({ +export const mavenSpec: PluginToolSpec = { name: "maven", - specs: { - darwin: spec, - linux: spec, - win32: spec, - }, -}) + description: "The Maven CLI.", + type: "binary", + builds: [ + { + platform: "darwin", + architecture: "amd64", + ...spec, + }, + { + platform: "linux", + architecture: "amd64", + ...spec, + }, + { + platform: "windows", + architecture: "amd64", + ...spec, + }, + ], +} diff --git a/garden-service/src/plugins/maven-container/openjdk.ts b/garden-service/src/plugins/maven-container/openjdk.ts index 6fa5736e518..77456f98f9b 100644 --- a/garden-service/src/plugins/maven-container/openjdk.ts +++ b/garden-service/src/plugins/maven-container/openjdk.ts @@ -6,7 +6,8 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { Library } from "../../util/ext-tools" +import { PluginToolSpec } from "../../types/plugin/tools" +import { posix } from "path" const jdk8Version = "jdk8u202-b08" const jdk11Version = "jdk-11.0.2+9" @@ -16,92 +17,116 @@ const jdk8Base = `https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/dow const jdk11Base = "https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.2%2B9/" const jdk13Base = "https://github.com/AdoptOpenJDK/openjdk13-binaries/releases/download/jdk-13%2B33/" -export const openJdks: { [version: number]: Library } = { - 8: new Library({ +export const openJdkSpecs: PluginToolSpec[] = [ + { name: "openjdk-8", - specs: { - darwin: { + description: "The OpenJDK 8 library.", + type: "library", + builds: [ + { + platform: "darwin", + architecture: "amd64", url: jdk8Base + "OpenJDK8U-jdk_x64_mac_hotspot_8u202b08.tar.gz", sha256: "059f7c18faa6722aa636bbd79bcdff3aee6a6da5b34940b072ea6e3af85bbe1d", extract: { format: "tar", - targetPath: [jdk8Version, "Contents", "Home"], + targetPath: posix.join(jdk8Version, "Contents", "Home"), }, }, - linux: { + { + platform: "linux", + architecture: "amd64", url: jdk8Base + "OpenJDK8U-jdk_x64_linux_hotspot_8u202b08.tar.gz", sha256: "f5a1c9836beb3ca933ec3b1d39568ecbb68bd7e7ca6a9989a21ff16a74d910ab", extract: { format: "tar", - targetPath: [jdk8Version], + targetPath: jdk8Version, }, }, - win32: { + { + platform: "windows", + architecture: "amd64", url: jdk8Base + "OpenJDK8U-jdk_x64_windows_hotspot_8u202b08.zip", sha256: "2637dab3bc81274e19991eebc27684276b482dd71d0f84fedf703d4fba3576e5", extract: { format: "zip", - targetPath: [jdk8Version], + targetPath: jdk8Version, }, }, - }, - }), - 11: new Library({ + ], + }, + { name: "openjdk-11", - specs: { - darwin: { + description: "The OpenJDK 11 library.", + type: "library", + builds: [ + { + platform: "darwin", + architecture: "amd64", url: jdk11Base + "OpenJDK11U-jdk_x64_mac_hotspot_11.0.2_9.tar.gz", sha256: "fffd4ed283e5cd443760a8ec8af215c8ca4d33ec5050c24c1277ba64b5b5e81a", extract: { format: "tar", - targetPath: [jdk11Version, "Contents", "Home"], + targetPath: posix.join(jdk11Version, "Contents", "Home"), }, }, - linux: { + { + platform: "linux", + architecture: "amd64", url: jdk11Base + "OpenJDK11U-jdk_x64_linux_hotspot_11.0.2_9.tar.gz", sha256: "d02089d834f7702ac1a9776d8d0d13ee174d0656cf036c6b68b9ffb71a6f610e", extract: { format: "tar", - targetPath: [jdk11Version], + targetPath: jdk11Version, }, }, - win32: { + { + platform: "windows", + architecture: "amd64", url: jdk11Base + "OpenJDK11U-jdk_x64_windows_hotspot_11.0.2_9.zip", sha256: "bde1648333abaf49c7175c9ee8ba9115a55fc160838ff5091f07d10c4bb50b3a", extract: { format: "zip", - targetPath: [jdk11Version], + targetPath: jdk11Version, }, }, - }, - }), - 13: new Library({ + ], + }, + { name: "openjdk-13", - specs: { - darwin: { + description: "The OpenJDK 13 library.", + type: "library", + builds: [ + { + platform: "darwin", + architecture: "amd64", url: jdk13Base + "OpenJDK13U-jdk_x64_mac_hotspot_13_33.tar.gz", sha256: "f948be96daba250b6695e22cb51372d2ba3060e4d778dd09c89548889783099f", extract: { format: "tar", - targetPath: [jdk13Version, "Contents", "Home"], + targetPath: posix.join(jdk13Version, "Contents", "Home"), }, }, - linux: { + { + platform: "linux", + architecture: "amd64", url: jdk13Base + "OpenJDK13U-jdk_x64_linux_hotspot_13_33.tar.gz", sha256: "e562caeffa89c834a69a44242d802eae3523875e427f07c05b1902c152638368", extract: { format: "tar", - targetPath: [jdk13Version], + targetPath: jdk13Version, }, }, - win32: { + { + platform: "windows", + architecture: "amd64", url: jdk13Base + "OpenJDK13U-jdk_x64_windows_hotspot_13_33.zip", sha256: "65d71a954167d538c7a260e64d9868ceffe60edd1108817a9c44fddf60d13569", extract: { format: "zip", - targetPath: [jdk13Version], + targetPath: jdk13Version, }, }, - }, - }), -} + ], + }, +] diff --git a/garden-service/src/plugins/openfaas/build.ts b/garden-service/src/plugins/openfaas/build.ts index 85cb1827df2..4aedace76b1 100644 --- a/garden-service/src/plugins/openfaas/build.ts +++ b/garden-service/src/plugins/openfaas/build.ts @@ -10,7 +10,6 @@ import { join } from "path" import { PrimitiveMap } from "../../config/common" import { KubernetesProvider } from "../kubernetes/config" import { dumpYaml } from "../../util/util" -import { faasCli } from "./faas-cli" import { BuildModuleParams } from "../../types/plugin/module/build" import { containerHelpers } from "../container/helpers" import { k8sBuildContainer, k8sGetContainerBuildStatus } from "../kubernetes/container/build" @@ -89,7 +88,7 @@ async function buildOpenfaasFunction( ) { await writeStackFile(provider, k8sProvider, module, {}) - return await faasCli.stdout({ + return await provider.tools["faas-cli"].stdout({ log, cwd: module.buildPath, args: ["build", "--shrinkwrap", "-f", stackFilename], diff --git a/garden-service/src/plugins/openfaas/config.ts b/garden-service/src/plugins/openfaas/config.ts index 7105227ba9d..635dc8e5b4b 100644 --- a/garden-service/src/plugins/openfaas/config.ts +++ b/garden-service/src/plugins/openfaas/config.ts @@ -17,8 +17,8 @@ import { Service } from "../../types/service" import { ExecModuleSpecBase, ExecTestSpec } from "../exec" import { KubernetesProvider } from "../kubernetes/config" import { CommonServiceSpec } from "../../config/service" -import { Provider, providerConfigBaseSchema, ProviderConfig } from "../../config/provider" -import { keyBy, union } from "lodash" +import { Provider, providerConfigBaseSchema, ProviderConfig, ProviderMap } from "../../config/provider" +import { union } from "lodash" import { ContainerModule } from "../container/config" import { ConfigureModuleParams, ConfigureModuleResult } from "../../types/plugin/module/configure" import { getNamespace } from "../kubernetes/namespace" @@ -140,10 +140,9 @@ export const configSchema = () => export type OpenFaasProvider = Provider export type OpenFaasPluginContext = PluginContext -export function getK8sProvider(providers: Provider[]): KubernetesProvider { - const providerMap = keyBy(providers, "name") +export function getK8sProvider(providers: ProviderMap): KubernetesProvider { // FIXME: use new plugin inheritance mechanism here, instead of explicitly checking for local-kubernetes - const provider = (providerMap["local-kubernetes"] || providerMap.kubernetes) + const provider = (providers["local-kubernetes"] || providers.kubernetes) if (!provider) { throw new ConfigurationError(`openfaas requires a kubernetes (or local-kubernetes) provider to be configured`, { diff --git a/garden-service/src/plugins/openfaas/faas-cli.ts b/garden-service/src/plugins/openfaas/faas-cli.ts index 0823d4da27b..d9c9a4372ef 100644 --- a/garden-service/src/plugins/openfaas/faas-cli.ts +++ b/garden-service/src/plugins/openfaas/faas-cli.ts @@ -6,22 +6,30 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { BinaryCmd } from "../../util/ext-tools" +import { PluginToolSpec } from "../../types/plugin/tools" -export const faasCli = new BinaryCmd({ +export const faasCliSpec: PluginToolSpec = { name: "faas-cli", - specs: { - darwin: { + description: "The faas-cli command line tool.", + type: "binary", + builds: [ + { + platform: "darwin", + architecture: "amd64", url: "https://github.com/openfaas/faas-cli/releases/download/0.9.5/faas-cli-darwin", sha256: "28beff63ef8234c1c937b14fd63e8c25244432897830650b8f76897fe4e22cbb", }, - linux: { + { + platform: "linux", + architecture: "amd64", url: "https://github.com/openfaas/faas-cli/releases/download/0.9.5/faas-cli", sha256: "f4c8014d953f42e0c83628c089aff36aaf306f9f1aea62e5f22c84ab4269d1f7", }, - win32: { + { + platform: "windows", + architecture: "amd64", url: "https://github.com/openfaas/faas-cli/releases/download/0.9.5/faas-cli.exe", sha256: "45d09e4dbff679c32aff8f86cc39e12c3687b6b344a9a20510c6c61f4e141eb5", }, - }, -}) + ], +} diff --git a/garden-service/src/plugins/openfaas/openfaas.ts b/garden-service/src/plugins/openfaas/openfaas.ts index 4d0266e5564..ddd53617b84 100644 --- a/garden-service/src/plugins/openfaas/openfaas.ts +++ b/garden-service/src/plugins/openfaas/openfaas.ts @@ -16,7 +16,7 @@ import { KubeApi } from "../kubernetes/api" import { waitForResources } from "../kubernetes/status/status" import { checkWorkloadStatus } from "../kubernetes/status/workload" import { createGardenPlugin } from "../../types/plugin/plugin" -import { faasCli } from "./faas-cli" +import { faasCliSpec } from "./faas-cli" import { getAllLogs } from "../kubernetes/logs" import { DeployServiceParams } from "../../types/plugin/service/deployService" import { GetServiceStatusParams } from "../../types/plugin/service/getServiceStatus" @@ -44,7 +44,7 @@ import { import { getOpenfaasModuleBuildStatus, buildOpenfaasModule, writeStackFile, stackFilename } from "./build" import { dedent } from "../../util/string" import { LogEntry } from "../../logger/log-entry" -import { Provider } from "../../config/provider" +import { ProviderMap } from "../../config/provider" import { parse } from "url" import { trim } from "lodash" import { getModuleTypeUrl, getGitHubUrl } from "../../docs/common" @@ -89,6 +89,7 @@ export const gardenPlugin = createGardenPlugin({ }, }, ], + tools: [faasCliSpec], }) const templateModuleConfig: ExecModuleConfig = { @@ -223,7 +224,7 @@ async function getFunctionNamespace( log: LogEntry, projectName: string, config: OpenFaasConfig, - dependencies: Provider[] + dependencies: ProviderMap ) { // Check for configured namespace in faas-netes custom values return ( @@ -270,7 +271,7 @@ async function deployService(params: DeployServiceParams): Promi while (true) { try { - await faasCli.stdout({ + await ctx.provider.tools["faas-cli"].stdout({ log, cwd: module.buildPath, args: ["deploy", "-f", stackFilename], @@ -331,7 +332,7 @@ async function deleteService(params: DeleteServiceParams): Promi found = !!status.state - await faasCli.stdout({ + await ctx.provider.tools["faas-cli"].stdout({ log, cwd: service.module.buildPath, args: ["remove", "-f", stackFilename], diff --git a/garden-service/src/plugins/terraform/cli.ts b/garden-service/src/plugins/terraform/cli.ts index 27b9652c9e9..9e6114287c9 100644 --- a/garden-service/src/plugins/terraform/cli.ts +++ b/garden-service/src/plugins/terraform/cli.ts @@ -6,11 +6,13 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { BinaryCmd, LibraryExtractSpec } from "../../util/ext-tools" import { ConfigurationError } from "../../exceptions" +import { PluginToolSpec } from "../../types/plugin/tools" +import { TerraformProvider } from "./terraform" -export function terraform(version: string) { - const cli = terraformClis[version] +export function terraform(provider: TerraformProvider) { + const version = provider.config.version + const cli = provider.tools["terraform-" + version.replace(/\./g, "-")] if (!cli) { throw new ConfigurationError(`Unsupported Terraform version: ${version}`, { @@ -22,95 +24,158 @@ export function terraform(version: string) { return cli } -const extract: LibraryExtractSpec = { - format: "zip", - targetPath: ["terraform"], -} - -export const terraformClis: { [version: string]: BinaryCmd } = { - "0.11.14": new BinaryCmd({ - name: "terraform-0.11.14", - specs: { - darwin: { +export const terraformCliSpecs: { [version: string]: PluginToolSpec } = { + "0.11.14": { + name: "terraform-0-11-14", + description: "The terraform CLI, v0.11.14", + type: "binary", + builds: [ + { + platform: "darwin", + architecture: "amd64", url: "https://releases.hashicorp.com/terraform/0.11.14/terraform_0.11.14_darwin_amd64.zip", sha256: "829bdba148afbd61eab4aafbc6087838f0333d8876624fe2ebc023920cfc2ad5", - extract, + extract: { + format: "zip", + targetPath: "terraform", + }, }, - linux: { + { + platform: "linux", + architecture: "amd64", url: "https://releases.hashicorp.com/terraform/0.11.14/terraform_0.11.14_linux_amd64.zip", sha256: "9b9a4492738c69077b079e595f5b2a9ef1bc4e8fb5596610f69a6f322a8af8dd", - extract, + extract: { + format: "zip", + targetPath: "terraform", + }, }, - win32: { + { + platform: "windows", + architecture: "amd64", url: "https://releases.hashicorp.com/terraform/0.11.14/terraform_0.11.14_windows_amd64.zip", sha256: "bfec66e2ad079a1fab6101c19617a82ef79357dc1b92ddca80901bb8d5312dc0", - extract, + extract: { + format: "zip", + targetPath: "terraform.exe", + }, }, - }, - }), - "0.12.7": new BinaryCmd({ - name: "terraform-0.12.7", - specs: { - darwin: { + ], + }, + "0.12.7": { + name: "terraform-0-12-7", + description: "The terraform CLI, v0.12.7", + type: "binary", + builds: [ + { + platform: "darwin", + architecture: "amd64", url: "https://releases.hashicorp.com/terraform/0.12.7/terraform_0.12.7_darwin_amd64.zip", sha256: "5cb59cdc4a8c4ebdfc0b8715936110e707d869c59603d27020e33b2be2e50f21", - extract, + extract: { + format: "zip", + targetPath: "terraform", + }, }, - linux: { + { + platform: "linux", + architecture: "amd64", url: "https://releases.hashicorp.com/terraform/0.12.7/terraform_0.12.7_linux_amd64.zip", sha256: "a0fa11217325f76bf1b4f53b0f7a6efb1be1826826ef8024f2f45e60187925e7", - extract, + extract: { + format: "zip", + targetPath: "terraform", + }, }, - win32: { + { + platform: "windows", + architecture: "amd64", url: "https://releases.hashicorp.com/terraform/0.12.7/terraform_0.12.7_windows_amd64.zip", sha256: "ce5b0eae0b443cbbb7c592d1b48bad6c8a3c5298932d35a4ebcba800c3488e4e", - extract, + extract: { + format: "zip", + targetPath: "terraform.exe", + }, }, - }, - }), - "0.12.21": new BinaryCmd({ - name: "terraform-0.12.21", - specs: { - darwin: { + ], + }, + "0.12.21": { + name: "terraform-0-12-21", + description: "The terraform CLI, v0.12.21", + type: "binary", + builds: [ + { + platform: "darwin", + architecture: "amd64", url: "https://releases.hashicorp.com/terraform/0.12.21/terraform_0.12.21_darwin_amd64.zip", sha256: "f89b620e59439fccc80950bbcbd37a069101cbef7029029a12227eee831e463f", - extract, + extract: { + format: "zip", + targetPath: "terraform", + }, }, - linux: { + { + platform: "linux", + architecture: "amd64", url: "https://releases.hashicorp.com/terraform/0.12.21/terraform_0.12.21_linux_amd64.zip", sha256: "ca0d0796c79d14ee73a3d45649dab5e531f0768ee98da71b31e423e3278e9aa9", - extract, + extract: { + format: "zip", + targetPath: "terraform", + }, }, - win32: { + { + platform: "windows", + architecture: "amd64", url: "https://releases.hashicorp.com/terraform/0.12.21/terraform_0.12.21_windows_amd64.zip", sha256: "254e5f870efe9d86a3f211a1b9c3c01325fc380e428f54542b7750d8bfd62bb1", - extract, + extract: { + format: "zip", + targetPath: "terraform.exe", + }, }, - }, - }), - "0.12.24": new BinaryCmd({ - name: "terraform-0.12.24", - specs: { - darwin: { + ], + }, + "0.12.24": { + name: "terraform-0-12-24", + description: "The terraform CLI, v0.12.24", + type: "binary", + builds: [ + { + platform: "darwin", + architecture: "amd64", url: "https://releases.hashicorp.com/terraform/0.12.24/terraform_0.12.24_darwin_amd64.zip", sha256: "72482000a5e25c33e88e95d70208304acfd09bf855a7ede110da032089d13b4f", - extract, + extract: { + format: "zip", + targetPath: "terraform", + }, }, - linux: { + { + platform: "linux", + architecture: "amd64", url: "https://releases.hashicorp.com/terraform/0.12.24/terraform_0.12.24_linux_amd64.zip", sha256: "602d2529aafdaa0f605c06adb7c72cfb585d8aa19b3f4d8d189b42589e27bf11", - extract, + extract: { + format: "zip", + targetPath: "terraform", + }, }, - win32: { + { + platform: "windows", + architecture: "amd64", url: "https://releases.hashicorp.com/terraform/0.12.24/terraform_0.12.24_windows_amd64.zip", sha256: "fd1679999d4555639f7074ad0a86d1a627a6da5e52dacdb77d3ecbcd1b5bca0a", - extract, + extract: { + format: "zip", + targetPath: "terraform.exe", + }, }, - }, - }), + ], + }, } -export const supportedVersions = Object.keys(terraformClis) +export const supportedVersions = Object.keys(terraformCliSpecs) // Default to latest Terraform version export const defaultTerraformVersion = supportedVersions[supportedVersions.length - 1] diff --git a/garden-service/src/plugins/terraform/commands.ts b/garden-service/src/plugins/terraform/commands.ts index b3d43c901b5..15b4e060a1f 100644 --- a/garden-service/src/plugins/terraform/commands.ts +++ b/garden-service/src/plugins/terraform/commands.ts @@ -46,7 +46,7 @@ function makeRootCommand(commandName: string) { await tfValidate(log, provider, root, provider.config.variables) args = [commandName, ...(await prepareVariables(root, provider.config.variables)), ...args] - await terraform(provider.config.version).spawnAndWait({ + await terraform(provider).spawnAndWait({ log, args, cwd: root, @@ -80,7 +80,7 @@ function makeModuleCommand(commandName: string) { await tfValidate(log, provider, root, provider.config.variables) args = [commandName, ...(await prepareVariables(root, module.spec.variables)), ...args.slice(1)] - await terraform(module.spec.version).spawnAndWait({ + await terraform(provider).spawnAndWait({ log, args, cwd: root, diff --git a/garden-service/src/plugins/terraform/common.ts b/garden-service/src/plugins/terraform/common.ts index 4a789a9bf25..1cd4404807c 100644 --- a/garden-service/src/plugins/terraform/common.ts +++ b/garden-service/src/plugins/terraform/common.ts @@ -31,9 +31,8 @@ export interface TerraformBaseSpec { export async function tfValidate(log: LogEntry, provider: TerraformProvider, root: string, variables: object) { const args = ["validate", "-json", ...(await prepareVariables(root, variables))] - const tfVersion = provider.config.version - const res = await terraform(tfVersion).json({ + const res = await terraform(provider).json({ log, args, ignoreError: true, @@ -46,9 +45,9 @@ export async function tfValidate(log: LogEntry, provider: TerraformProvider, roo if (reasons.includes("Could not satisfy plugin requirements") || reasons.includes("Module not installed")) { // We need to run `terraform init` and retry validation log.debug("Initializing Terraform") - await terraform(tfVersion).exec({ log, args: ["init"], cwd: root, timeout: 300 }) + await terraform(provider).exec({ log, args: ["init"], cwd: root, timeout: 300 }) - const retryRes = await terraform(tfVersion).json({ + const retryRes = await terraform(provider).json({ log, args, ignoreError: true, @@ -63,12 +62,13 @@ export async function tfValidate(log: LogEntry, provider: TerraformProvider, roo } } -export async function getTfOutputs(log: LogEntry, terraformVersion: string, workingDir: string) { - const res = await terraform(terraformVersion).json({ +export async function getTfOutputs(log: LogEntry, provider: TerraformProvider, workingDir: string) { + const res = await terraform(provider).json({ log, args: ["output", "-json"], cwd: workingDir, }) + return mapValues(res, (v: any) => v.value) } @@ -106,11 +106,10 @@ export async function getStackStatus({ variables, }: GetTerraformStackStatusParams): Promise { await tfValidate(log, provider, root, variables) - const tfVersion = provider.config.version const logEntry = log.verbose({ section: "terraform", msg: "Running plan...", status: "active" }) - const plan = await terraform(tfVersion).exec({ + const plan = await terraform(provider).exec({ log, ignoreError: true, args: [ @@ -151,18 +150,18 @@ export async function getStackStatus({ export async function applyStack({ log, + provider, root, variables, - version, }: { log: LogEntry + provider: TerraformProvider root: string variables: object - version: string }) { const args = ["apply", "-auto-approve", "-input=false", ...(await prepareVariables(root, variables))] - const proc = await terraform(version).spawn({ log, args, cwd: root }) + const proc = await terraform(provider).spawn({ log, args, cwd: root }) const statusLine = log.info("→ Applying Terraform stack...") const logStream = split2() diff --git a/garden-service/src/plugins/terraform/init.ts b/garden-service/src/plugins/terraform/init.ts index ce2111385ea..10c8a5ea3ad 100644 --- a/garden-service/src/plugins/terraform/init.ts +++ b/garden-service/src/plugins/terraform/init.ts @@ -24,12 +24,11 @@ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusPar const autoApply = provider.config.autoApply const root = getRoot(ctx, provider) const variables = provider.config.variables - const tfVersion = provider.config.version const status = await getStackStatus({ log, provider, root, variables }) if (status === "up-to-date") { - const outputs = await getTfOutputs(log, tfVersion, root) + const outputs = await getTfOutputs(log, provider, root) return { ready: true, outputs } } else if (status === "outdated") { if (autoApply) { @@ -42,7 +41,7 @@ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusPar ${chalk.white.bold("garden plugins terraform apply-root")} to make sure the stack is in the intended state. `), }) - const outputs = await getTfOutputs(log, tfVersion, root) + const outputs = await getTfOutputs(log, provider, root) return { ready: true, outputs } } } else { @@ -58,15 +57,14 @@ export async function prepareEnvironment({ ctx, log }: PrepareEnvironmentParams) return { status: { ready: true, outputs: {} } } } - const tfVersion = provider.config.version const root = getRoot(ctx, provider) // Don't run apply when running plugin commands if (provider.config.autoApply && !(ctx.command?.name === "plugins" && ctx.command?.args.plugin === provider.name)) { - await applyStack({ log, root, variables: provider.config.variables, version: provider.config.version }) + await applyStack({ log, provider, root, variables: provider.config.variables }) } - const outputs = await getTfOutputs(log, tfVersion, root) + const outputs = await getTfOutputs(log, provider, root) return { status: { diff --git a/garden-service/src/plugins/terraform/module.ts b/garden-service/src/plugins/terraform/module.ts index 545b8a89947..22853376e2a 100644 --- a/garden-service/src/plugins/terraform/module.ts +++ b/garden-service/src/plugins/terraform/module.ts @@ -117,7 +117,7 @@ export async function getTerraformStatus({ return { state: status === "up-to-date" ? "ready" : "outdated", version: module.version.versionString, - outputs: await getTfOutputs(log, provider.config.version, root), + outputs: await getTfOutputs(log, provider, root), detail: {}, } } @@ -131,7 +131,7 @@ export async function deployTerraform({ const root = getModuleStackRoot(module) if (module.spec.autoApply) { - await applyStack({ log, root, variables: module.spec.variables, version: module.spec.version }) + await applyStack({ log, provider, root, variables: module.spec.variables }) } else { const templateKey = `\${runtime.services.${module.name}.outputs.*}` log.warn( @@ -148,7 +148,7 @@ export async function deployTerraform({ return { state: "ready", version: module.version.versionString, - outputs: await getTfOutputs(log, provider.config.version, root), + outputs: await getTfOutputs(log, provider, root), detail: {}, } } diff --git a/garden-service/src/plugins/terraform/terraform.ts b/garden-service/src/plugins/terraform/terraform.ts index 7f335bc15d8..d7d2789a3ea 100644 --- a/garden-service/src/plugins/terraform/terraform.ts +++ b/garden-service/src/plugins/terraform/terraform.ts @@ -13,7 +13,7 @@ import { getEnvironmentStatus, prepareEnvironment } from "./init" import { providerConfigBaseSchema, ProviderConfig, Provider } from "../../config/provider" import { joi } from "../../config/common" import { dedent } from "../../util/string" -import { supportedVersions, defaultTerraformVersion } from "./cli" +import { supportedVersions, defaultTerraformVersion, terraformCliSpecs } from "./cli" import { ConfigureProviderParams, ConfigureProviderResult } from "../../types/plugin/provider/configureProvider" import { ConfigurationError } from "../../exceptions" import { variablesSchema, TerraformBaseSpec } from "./common" @@ -120,6 +120,7 @@ export const gardenPlugin = createGardenPlugin({ }, }, ], + tools: Object.values(terraformCliSpecs), }) async function configureProvider({ diff --git a/garden-service/src/process.ts b/garden-service/src/process.ts index 2a6dcb614a6..1ba0f098fb5 100644 --- a/garden-service/src/process.ts +++ b/garden-service/src/process.ts @@ -68,12 +68,14 @@ export async function processModules({ log.info(renderDivider()) } + let statusLine: LogEntry + if (watch && !!footerLog) { - footerLog.info("") + statusLine = footerLog.info("").placeholder() garden.events.on("taskGraphProcessing", () => { - const emoji = printEmoji("hourglass_flowing_sand", footerLog) - footerLog.setState(`${emoji} Processing...`) + const emoji = printEmoji("hourglass_flowing_sand", statusLine) + statusLine.setState(`${emoji} Processing...`) }) } @@ -97,8 +99,8 @@ export async function processModules({ await garden.startWatcher(graph) const waiting = () => { - if (!!footerLog) { - footerLog.setState({ emoji: "clock2", msg: chalk.gray("Waiting for code changes...") }) + if (!!statusLine) { + statusLine.setState({ emoji: "clock2", msg: chalk.gray("Waiting for code changes...") }) } garden.events.emit("watchingForChanges", {}) diff --git a/garden-service/src/tasks/resolve-module.ts b/garden-service/src/tasks/resolve-module.ts index 538c34a86e9..e39695df6c5 100644 --- a/garden-service/src/tasks/resolve-module.ts +++ b/garden-service/src/tasks/resolve-module.ts @@ -17,7 +17,7 @@ import { keyBy, fromPairs } from "lodash" import { ConfigurationError } from "../exceptions" import { RuntimeContext } from "../runtime-context" import { ModuleConfigContext } from "../config/config-context" -import { Provider } from "../config/provider" +import { ProviderMap } from "../config/provider" import { resolveModuleConfig } from "../resolve-module" import { getModuleTemplateReferences } from "../template-string" import { Profile } from "../util/profiling" @@ -26,7 +26,7 @@ interface ResolveModuleConfigTaskParams { garden: Garden log: LogEntry moduleConfig: ModuleConfig - resolvedProviders: Provider[] + resolvedProviders: ProviderMap runtimeContext?: RuntimeContext } @@ -39,7 +39,7 @@ export class ResolveModuleConfigTask extends BaseTask { type: TaskType = "resolve-module-config" private moduleConfig: ModuleConfig - private resolvedProviders: Provider[] + private resolvedProviders: ProviderMap private runtimeContext?: RuntimeContext constructor({ garden, log, moduleConfig, resolvedProviders, runtimeContext }: ResolveModuleConfigTaskParams) { @@ -127,7 +127,7 @@ interface ResolveModuleTaskParams { garden: Garden log: LogEntry moduleConfig: ModuleConfig - resolvedProviders: Provider[] + resolvedProviders: ProviderMap runtimeContext?: RuntimeContext } @@ -139,7 +139,7 @@ export class ResolveModuleTask extends BaseTask { type: TaskType = "resolve-module" private moduleConfig: ModuleConfig - private resolvedProviders: Provider[] + private resolvedProviders: ProviderMap private runtimeContext?: RuntimeContext constructor({ garden, log, moduleConfig, resolvedProviders, runtimeContext }: ResolveModuleTaskParams) { diff --git a/garden-service/src/tasks/resolve-provider.ts b/garden-service/src/tasks/resolve-provider.ts index 4c1311bdc90..96230f9c6f8 100644 --- a/garden-service/src/tasks/resolve-provider.ts +++ b/garden-service/src/tasks/resolve-provider.ts @@ -8,7 +8,13 @@ import chalk from "chalk" import { BaseTask, TaskParams, TaskType } from "./base" -import { ProviderConfig, Provider, providerFromConfig, getProviderTemplateReferences } from "../config/provider" +import { + ProviderConfig, + Provider, + providerFromConfig, + getProviderTemplateReferences, + ProviderMap, +} from "../config/provider" import { resolveTemplateStrings } from "../template-string" import { ConfigurationError, PluginError } from "../exceptions" import { keyBy, omit, flatten } from "lodash" @@ -22,6 +28,7 @@ import Bluebird from "bluebird" import { defaultEnvironmentStatus } from "../types/plugin/provider/getEnvironmentStatus" import { getPluginBases, getPluginBaseNames } from "../plugins" import { Profile } from "../util/profiling" +import { PluginTool } from "../util/ext-tools" interface Params extends TaskParams { plugin: GardenPlugin @@ -104,7 +111,10 @@ export class ResolveProviderTask extends BaseTask { } async process(dependencyResults: TaskResults) { - const resolvedProviders: Provider[] = Object.values(dependencyResults).map((result) => result && result.output) + const resolvedProviders: ProviderMap = keyBy( + Object.values(dependencyResults).map((result) => result && result.output), + "name" + ) // Return immediately if the provider has been previously resolved const alreadyResolvedProviders = this.garden["resolvedProviders"][this.config.name] @@ -146,6 +156,11 @@ export class ResolveProviderTask extends BaseTask { const actions = await this.garden.getActionRouter() + const tools = keyBy( + (this.plugin.tools || []).map((spec) => new PluginTool(spec)), + "name" + ) + const configureOutput = await actions.configureProvider({ environmentName: this.garden.environmentName, namespace: this.garden.namespace, @@ -156,6 +171,7 @@ export class ResolveProviderTask extends BaseTask { projectName: this.garden.projectName, projectRoot: this.garden.projectRoot, dependencies: resolvedProviders, + tools, }) this.log.silly(`Validating ${providerName} config returned from configureProvider handler`) @@ -191,10 +207,16 @@ export class ResolveProviderTask extends BaseTask { this.log.silly(`Ensuring ${providerName} provider is ready`) - const tmpProvider = providerFromConfig(resolvedConfig, resolvedProviders, moduleConfigs, defaultEnvironmentStatus) + const tmpProvider = providerFromConfig( + resolvedConfig, + resolvedProviders, + moduleConfigs, + defaultEnvironmentStatus, + tools + ) const status = await this.ensurePrepared(tmpProvider) - return providerFromConfig(resolvedConfig, resolvedProviders, moduleConfigs, status) + return providerFromConfig(resolvedConfig, resolvedProviders, moduleConfigs, status, tools) } private async ensurePrepared(tmpProvider: Provider) { diff --git a/garden-service/src/types/plugin/plugin.ts b/garden-service/src/types/plugin/plugin.ts index d5dd745ac2e..b147d7e7349 100644 --- a/garden-service/src/types/plugin/plugin.ts +++ b/garden-service/src/types/plugin/plugin.ts @@ -43,6 +43,7 @@ import { StopPortForwardParams, stopPortForward } from "./service/stopPortForwar import { AugmentGraphResult, AugmentGraphParams, augmentGraph } from "./provider/augmentGraph" import { suggestModules, SuggestModulesParams, SuggestModulesResult } from "./module/suggestModules" import { templateStringLiteral } from "../../docs/common" +import { toolSchema, PluginToolSpec } from "./tools" export interface ActionHandlerParamsBase { base?: ActionHandler @@ -361,6 +362,7 @@ interface GardenPluginSpec { handlers?: Partial commands?: PluginCommand[] + tools?: PluginToolSpec[] createModuleTypes?: ModuleTypeDefinition[] extendModuleTypes?: ModuleTypeExtension[] @@ -540,6 +542,19 @@ export const pluginSchema = () => .unique("name").description(dedent` List of module types to extend/override with additional handlers. `), + + tools: joi + .array() + .items(toolSchema()) + .unique("name").description(dedent` + List of tools that this plugin exposes via \`garden tools \`, and within its own plugin handlers and commands. + + The tools are downloaded automatically on first use, and cached under the user's global \`~/.garden\` directory. + + If multiple plugins specify a tool with the same name, you can reference them prefixed with the plugin name and a period, e.g. \`kubernetes.kubectl\` to pick a specific plugin's command. Otherwise a warning is emitted when running \`garden tools\`, and the tool that's configured by the plugin that is last in the dependency order is used. Since that can often be ambiguous, it is highly recommended to use the fully qualified name in automated scripts. + + If you specify a \`base\`, new tools are added in addition to the tools of the base plugin, and if you specify a tool with the same name as one in the base plugin, you override the one declared in the base. + `), }) .description("The schema for Garden plugins.") diff --git a/garden-service/src/types/plugin/provider/augmentGraph.ts b/garden-service/src/types/plugin/provider/augmentGraph.ts index d7b1092807c..a8dfec25fff 100644 --- a/garden-service/src/types/plugin/provider/augmentGraph.ts +++ b/garden-service/src/types/plugin/provider/augmentGraph.ts @@ -8,14 +8,14 @@ import { PluginActionParamsBase, actionParamsSchema } from "../base" import { dedent } from "../../../util/string" -import { joi, joiArray, joiIdentifier } from "../../../config/common" +import { joi, joiArray, joiIdentifier, joiIdentifierMap } from "../../../config/common" import { baseModuleSpecSchema, AddModuleSpec } from "../../../config/module" -import { Provider, providerSchema } from "../../../config/provider" +import { providerSchema, ProviderMap } from "../../../config/provider" import { Module, moduleSchema } from "../../module" export interface AugmentGraphParams extends PluginActionParamsBase { modules: Module[] - providers: Provider[] + providers: ProviderMap } interface AddDependency { @@ -51,7 +51,7 @@ export const augmentGraph = () => ({ handlers defined by other providers that this provider depends on. ` ), - providers: joiArray(providerSchema()).description("All configured providers in the project."), + providers: joiIdentifierMap(providerSchema()).description("Map of all configured providers in the project."), }), resultSchema: joi.object().keys({ addBuildDependencies: joi diff --git a/garden-service/src/types/plugin/provider/configureProvider.ts b/garden-service/src/types/plugin/provider/configureProvider.ts index 81a42003f91..deeb8ff97f1 100644 --- a/garden-service/src/types/plugin/provider/configureProvider.ts +++ b/garden-service/src/types/plugin/provider/configureProvider.ts @@ -7,25 +7,27 @@ */ import { projectNameSchema, projectRootSchema } from "../../../config/project" -import { ProviderConfig, Provider, providerConfigBaseSchema, providerSchema } from "../../../config/provider" +import { ProviderConfig, providerConfigBaseSchema, providerSchema, ProviderMap } from "../../../config/provider" import { logEntrySchema } from "../base" import { configStoreSchema, ConfigStore } from "../../../config-store" -import { joiArray, joi, joiIdentifier } from "../../../config/common" +import { joiArray, joi, joiIdentifier, joiIdentifierMap } from "../../../config/common" import { moduleConfigSchema, ModuleConfig } from "../../../config/module" import { deline, dedent } from "../../../util/string" import { ActionHandler, ActionHandlerParamsBase } from "../plugin" import { LogEntry } from "../../../logger/log-entry" +import { PluginTools } from "../tools" // Note: These are the only plugin handler params that don't inherit from PluginActionParamsBase export interface ConfigureProviderParams extends ActionHandlerParamsBase { - log: LogEntry config: T + configStore: ConfigStore + dependencies: ProviderMap environmentName: string + log: LogEntry namespace?: string projectName: string projectRoot: string - dependencies: Provider[] - configStore: ConfigStore + tools: PluginTools base?: ActionHandler, ConfigureProviderResult> } @@ -53,8 +55,9 @@ export const configureProvider = () => ({ log: logEntrySchema(), projectName: projectNameSchema(), projectRoot: projectRootSchema(), - dependencies: joiArray(providerSchema()).description("All providers that this provider depends on."), + dependencies: joiIdentifierMap(providerSchema()).description("Map of all providers that this provider depends on."), configStore: configStoreSchema(), + tools: joiIdentifierMap(joi.object()), }), resultSchema: joi.object().keys({ config: providerConfigBaseSchema(), diff --git a/garden-service/src/types/plugin/tools.ts b/garden-service/src/types/plugin/tools.ts new file mode 100644 index 00000000000..17b19a96234 --- /dev/null +++ b/garden-service/src/types/plugin/tools.ts @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { joi, joiIdentifier } from "../../config/common" +import { deline } from "../../util/string" +import { PluginTool } from "../../util/ext-tools" + +export interface ToolBuildSpec { + platform: string + architecture: string + url: string + sha256: string + extract?: { + format: string + targetPath: string + } +} + +const toolBuildSchema = () => + joi.object().keys({ + platform: joi + .string() + .allow("darwin", "linux", "windows") + .required() + .example("linux") + .description("The platform this build is for."), + architecture: joi + .string() + .allow("amd64") + .required() + .example("amd64") + .description("The architecture of the build."), + url: joi + .string() + .uri({ allowRelative: false }) + .required() + .example("https://github.com/some/tool/releases/download/my-tool-linux-amd64.tar.gz") + .description("The URL to download for the build."), + sha256: joi + .string() + .required() + .example("a81b23abe67e70f8395ff7a3659bea6610fba98cda1126ef19e0a995f0075d54") + .description("The SHA256 sum the target URL should have."), + extract: joi + .object() + .keys({ + format: joi + .string() + .allow("tar", "zip") + .required() + .example("tar") + .description("The archive format."), + targetPath: joi + .posixPath() + .relativeOnly() + .example("my-tool/binary.exe") + .description("The path to the binary within the archive, if applicable."), + }) + .description("Specify instructions for extraction, if the URL points to an archive."), + }) + +export interface PluginToolSpec { + name: string + description: string + type: "library" | "binary" + builds: ToolBuildSpec[] +} + +export interface PluginTools { + [name: string]: PluginTool +} + +export const toolSchema = () => + joi.object().keys({ + name: joiIdentifier().description("The name of the tool. This must be unique within the provider."), + description: joi + .string() + .required() + .description("A short description of the tool, used for help texts."), + type: joi + .string() + .allow("library", "binary") + .description( + `Set this to "library" if the tool is not an executable. Set to "binary" if it should be exposed as a command.` + ), + builds: joi + .array() + .items(toolBuildSchema()) + .required().description(deline` + List of platform and architecture builds, with URLs and (if applicable) archive extraction information. + The list should include at least an amd64 build for each of darwin, linux and windows. + `), + }) diff --git a/garden-service/src/util/ext-tools.ts b/garden-service/src/util/ext-tools.ts index 0f2c4d3a44d..cafc51153ac 100644 --- a/garden-service/src/util/ext-tools.ts +++ b/garden-service/src/util/ext-tools.ts @@ -7,12 +7,12 @@ */ import { platform } from "os" -import { pathExists, createWriteStream, ensureDir, chmod, remove, move } from "fs-extra" +import { pathExists, createWriteStream, ensureDir, chmod, remove, move, createReadStream } from "fs-extra" import { ConfigurationError, ParameterError, GardenBaseError } from "../exceptions" -import { join, dirname, basename, sep } from "path" -import { hashString, exec, uuidv4 } from "./util" +import { join, dirname, basename, posix } from "path" +import { hashString, exec, uuidv4, getPlatform, getArchitecture } from "./util" import tar from "tar" -import { SupportedPlatform, GARDEN_GLOBAL_PATH } from "../constants" +import { GARDEN_GLOBAL_PATH } from "../constants" import { LogEntry } from "../logger/log-entry" import { Extract } from "unzipper" import { createHash } from "crypto" @@ -20,80 +20,180 @@ import crossSpawn from "cross-spawn" import { spawn } from "./util" import { Writable } from "stream" import got from "got/dist/source" +import { PluginToolSpec, ToolBuildSpec } from "../types/plugin/tools" +import { parse } from "url" const AsyncLock = require("async-lock") const toolsPath = join(GARDEN_GLOBAL_PATH, "tools") -const defaultCommandTimeoutSecs = 60 * 10 - -export interface LibraryExtractSpec { - // Archive format. Note: the "tar" format also implicitly supports gzip and bz2 compression. - format: "tar" | "zip" - // Path to the target file or directory, relative to the download directory, after downloading and - // extracting the archive. For BinaryCmds, this should point to the executable in the archive. - targetPath: string[] + +export class DownloadError extends GardenBaseError { + type = "download" } -export interface LibraryPlatformSpec { - url: string - // Optionally specify sha256 checksum for validation. - sha256?: string - // If the URL contains an archive, provide extraction instructions. - extract?: LibraryExtractSpec +export interface ExecParams { + args?: string[] + cwd?: string + env?: { [key: string]: string } + log: LogEntry + timeout?: number + input?: Buffer | string + ignoreError?: boolean + stdout?: Writable + stderr?: Writable } -// TODO: support different architectures? (the Garden class currently errors on non-x64 archs, and many tools may -// only be available in x64). -interface LibrarySpec { - name: string - specs: { [key in SupportedPlatform]: LibraryPlatformSpec } +export interface SpawnParams extends ExecParams { + tty?: boolean + rawMode?: boolean // Only used if tty = true. See also: https://nodejs.org/api/tty.html#tty_readstream_setrawmode_mode } -export class DownloadError extends GardenBaseError { - type = "download" +interface PluginToolOpts { + platform?: string + architecture?: string } /** - * This helper class allows you to declare a library dependency by providing a URL to a file or an archive, - * for each of our supported platforms. When requesting the path to the library, the appropriate URL for the - * current platform will be downloaded, extracted (if applicable) and cached in the user's home directory + * This helper class allows you to declare a tool dependency by providing a URL to a single-file binary, + * or an archive containing an executable, for each of our supported platforms. When executing the tool, + * the appropriate URL for the current platform will be downloaded and cached in the user's home directory * (under .garden/tools//). * - * Note: The file or archive currently needs to be self-contained and work without further installation steps. + * Note: The binary or archive currently needs to be self-contained and work without further installation steps. */ -export class Library { +export class PluginTool { name: string - spec: LibraryPlatformSpec + type: string + spec: PluginToolSpec + buildSpec: ToolBuildSpec private lock: any private toolPath: string private versionDirname: string protected versionPath: string - protected targetSubpath: string[] + protected targetSubpath: string + private chmodDone: boolean - constructor(spec: LibrarySpec, currentPlatform = platform()) { - const platformSpec = spec.specs[currentPlatform] + constructor(spec: PluginToolSpec, opts: PluginToolOpts = {}) { + const _platform = opts.platform || getPlatform() + const architecture = opts.architecture || getArchitecture() - if (!platformSpec) { - throw new ConfigurationError(`Command ${spec.name} doesn't have a spec for this platform (${currentPlatform})`, { - spec, - currentPlatform, - }) + this.buildSpec = spec.builds.find((build) => build.platform === _platform && build.architecture === architecture)! + + if (!this.buildSpec) { + throw new ConfigurationError( + `Command ${spec.name} doesn't have a spec for this platform/architecture (${platform}-${architecture})`, + { + spec, + platform, + architecture, + } + ) } this.lock = new AsyncLock() this.name = spec.name - this.spec = platformSpec + this.type = spec.type + this.spec = spec this.toolPath = join(toolsPath, this.name) - this.versionDirname = hashString(this.spec.url, 16) + this.versionDirname = hashString(this.buildSpec.url, 16) this.versionPath = join(this.toolPath, this.versionDirname) - this.targetSubpath = this.spec.extract ? this.spec.extract.targetPath : [basename(this.spec.url)] + this.targetSubpath = this.buildSpec.extract ? this.buildSpec.extract.targetPath : basename(this.buildSpec.url) + this.chmodDone = false } async getPath(log: LogEntry) { await this.download(log) - return join(this.versionPath, ...this.targetSubpath) + const path = join(this.versionPath, ...this.targetSubpath.split(posix.sep)) + + if (this.spec.type === "binary") { + // Make sure the target path is executable + if (!this.chmodDone) { + await chmod(path, 0o755) + this.chmodDone = true + } + } + + return path + } + + async exec({ args, cwd, env, log, timeout, input, ignoreError, stdout, stderr }: ExecParams) { + const path = await this.getPath(log) + + if (!args) { + args = [] + } + if (!cwd) { + cwd = dirname(path) + } + + log.debug(`Execing '${path} ${args.join(" ")}' in ${cwd}`) + + return exec(path, args, { + cwd, + timeout: timeout ? timeout * 1000 : undefined, + env, + input, + reject: !ignoreError, + stdout, + stderr, + }) + } + + async stdout(params: ExecParams) { + try { + const res = await this.exec(params) + return res.stdout + } catch (err) { + // Add log output to error + if (err.all) { + err.message += "\n\n" + err.all + } + throw err + } + } + + async json(params: ExecParams) { + const out = await this.stdout(params) + return JSON.parse(out) + } + + async spawn({ args, cwd, env, log }: SpawnParams) { + const path = await this.getPath(log) + + if (!args) { + args = [] + } + if (!cwd) { + cwd = dirname(path) + } + + log.debug(`Spawning '${path} ${args.join(" ")}' in ${cwd}`) + return crossSpawn(path, args, { cwd, env }) + } + + async spawnAndWait({ args, cwd, env, log, ignoreError, rawMode, stdout, stderr, timeout, tty }: SpawnParams) { + const path = await this.getPath(log) + + if (!args) { + args = [] + } + if (!cwd) { + cwd = dirname(path) + } + + log.debug(`Spawning '${path} ${args.join(" ")}' in ${cwd}`) + return spawn(path, args || [], { + cwd, + timeout, + ignoreError, + env, + rawMode, + stdout, + stderr, + tty, + }) } protected async download(log: LogEntry) { @@ -103,22 +203,22 @@ export class Library { } const tmpPath = join(this.toolPath, this.versionDirname + "." + uuidv4().substr(0, 8)) - const targetAbsPath = join(tmpPath, ...this.targetSubpath) + const targetAbsPath = join(tmpPath, ...this.targetSubpath.split(posix.sep)) const logEntry = log.info({ - symbol: "info", + status: "active", msg: `Fetching ${this.name}...`, }) - const debug = logEntry.debug(`Downloading ${this.spec.url}...`) + const debug = logEntry.debug(`Downloading ${this.buildSpec.url}...`) await ensureDir(tmpPath) try { await this.fetch(tmpPath, log) - if (this.spec.extract && !(await pathExists(targetAbsPath))) { + if (this.buildSpec.extract && !(await pathExists(targetAbsPath))) { throw new ConfigurationError( - `Archive ${this.spec.url} does not contain a file or directory at ${this.targetSubpath.join(sep)}`, + `Archive ${this.buildSpec.url} does not contain a file or directory at ${this.targetSubpath}`, { name: this.name, spec: this.spec } ) } @@ -137,10 +237,16 @@ export class Library { } protected async fetch(tmpPath: string, log: LogEntry) { - const response = got.stream({ - method: "GET", - url: this.spec.url, - }) + const parsed = parse(this.buildSpec.url) + const protocol = parsed.protocol + + const response = + protocol === "file:" + ? createReadStream(parsed.path!) + : got.stream({ + method: "GET", + url: this.buildSpec.url, + }) // compute the sha256 checksum const hash = createHash("sha256") @@ -149,7 +255,7 @@ export class Library { return new Promise((resolve, reject) => { response.on("error", (err) => { - log.setError(`Failed fetching ${this.spec.url}`) + log.setError(`Failed fetching ${this.buildSpec.url}`) reject(err) }) @@ -162,9 +268,9 @@ export class Library { return } - if (this.spec.sha256 && sha256 !== this.spec.sha256) { + if (this.buildSpec.sha256 && sha256 !== this.buildSpec.sha256) { reject( - new DownloadError(`Invalid checksum from ${this.spec.url} (got ${sha256})`, { + new DownloadError(`Invalid checksum from ${this.buildSpec.url} (got ${sha256})`, { name: this.name, spec: this.spec, sha256, @@ -173,12 +279,12 @@ export class Library { } }) - if (!this.spec.extract) { - const targetExecutable = join(tmpPath, ...this.targetSubpath) + if (!this.buildSpec.extract) { + const targetExecutable = join(tmpPath, ...this.targetSubpath.split(posix.sep)) response.pipe(createWriteStream(targetExecutable)) response.on("end", () => resolve()) } else { - const format = this.spec.extract.format + const format = this.buildSpec.extract.format let extractor: Writable if (format === "tar") { @@ -203,145 +309,10 @@ export class Library { response.pipe(extractor) extractor.on("error", (err) => { - log.setError(`Failed extracting ${format} archive ${this.spec.url}`) + log.setError(`Failed extracting ${format} archive ${this.buildSpec.url}`) reject(err) }) } }) } } - -interface BinarySpec extends LibrarySpec { - defaultTimeout?: number -} - -export interface ExecParams { - args?: string[] - cwd?: string - env?: { [key: string]: string } - log: LogEntry - timeout?: number - input?: Buffer | string - ignoreError?: boolean - stdout?: Writable - stderr?: Writable -} - -export interface SpawnParams extends ExecParams { - tty?: boolean - rawMode?: boolean // Only used if tty = true. See also: https://nodejs.org/api/tty.html#tty_readstream_setrawmode_mode -} - -/** - * This helper class allows you to declare a tool dependency by providing a URL to a single-file binary, - * or an archive containing an executable, for each of our supported platforms. When executing the tool, - * the appropriate URL for the current platform will be downloaded and cached in the user's home directory - * (under .garden/tools//). - * - * Note: The binary or archive currently needs to be self-contained and work without further installation steps. - */ -export class BinaryCmd extends Library { - name: string - spec: LibraryPlatformSpec - - private chmodDone: boolean - private defaultTimeoutSecs: number - - constructor(spec: BinarySpec) { - super(spec) - this.chmodDone = false - this.defaultTimeoutSecs = spec.defaultTimeout || defaultCommandTimeoutSecs - } - - async getPath(log: LogEntry) { - const path = await super.getPath(log) - // Make sure the target path is executable - if (!this.chmodDone) { - await chmod(path, 0o755) - this.chmodDone = true - } - return path - } - - async exec({ args, cwd, env, log, timeout, input, ignoreError, stdout, stderr }: ExecParams) { - const path = await this.getPath(log) - - if (!args) { - args = [] - } - if (!cwd) { - cwd = dirname(path) - } - - log.debug(`Execing '${path} ${args.join(" ")}' in ${cwd}`) - - return exec(path, args, { - cwd, - timeout: this.getTimeout(timeout) * 1000, - env, - input, - reject: !ignoreError, - stdout, - stderr, - }) - } - - async stdout(params: ExecParams) { - try { - const res = await this.exec(params) - return res.stdout - } catch (err) { - // Add log output to error - if (err.all) { - err.message += "\n\n" + err.all - } - throw err - } - } - - async json(params: ExecParams) { - const out = await this.stdout(params) - return JSON.parse(out) - } - - async spawn({ args, cwd, env, log }: SpawnParams) { - const path = await this.getPath(log) - - if (!args) { - args = [] - } - if (!cwd) { - cwd = dirname(path) - } - - log.debug(`Spawning '${path} ${args.join(" ")}' in ${cwd}`) - return crossSpawn(path, args, { cwd, env }) - } - - async spawnAndWait({ args, cwd, env, log, ignoreError, rawMode, stdout, stderr, timeout, tty }: SpawnParams) { - const path = await this.getPath(log) - - if (!args) { - args = [] - } - if (!cwd) { - cwd = dirname(path) - } - - log.debug(`Spawning '${path} ${args.join(" ")}' in ${cwd}`) - return spawn(path, args || [], { - cwd, - timeout: this.getTimeout(timeout), - ignoreError, - env, - rawMode, - stdout, - stderr, - tty, - }) - } - - private getTimeout(timeout?: number) { - return timeout === undefined ? this.defaultTimeoutSecs : timeout - } -} diff --git a/garden-service/src/util/util.ts b/garden-service/src/util/util.ts index dccbef92aa1..01750267f3c 100644 --- a/garden-service/src/util/util.ts +++ b/garden-service/src/util/util.ts @@ -575,3 +575,22 @@ export function pushToKey(obj: object, key: string, value: any) { export function isPromise(obj: any): obj is Promise { return !!obj && (typeof obj === "object" || typeof obj === "function") && typeof obj.then === "function" } + +// Used to make the platforms more consistent with other tools +const platformMap = { + win32: "windows", +} + +const archMap = { + x32: "386", + x64: "amd64", +} + +export function getPlatform() { + return platformMap[process.platform] || process.platform +} + +export function getArchitecture() { + const arch = process.arch + return archMap[arch] || arch +} diff --git a/garden-service/test/data/tools/tool-a.sh b/garden-service/test/data/tools/tool-a.sh new file mode 100644 index 00000000000..a5f6045fe33 --- /dev/null +++ b/garden-service/test/data/tools/tool-a.sh @@ -0,0 +1,3 @@ +#!/bin/sh +echo test-a +exit $1 diff --git a/garden-service/test/data/tools/tool-b.sh b/garden-service/test/data/tools/tool-b.sh new file mode 100644 index 00000000000..023b6808108 --- /dev/null +++ b/garden-service/test/data/tools/tool-b.sh @@ -0,0 +1,3 @@ +#!/bin/sh +echo test-b +exit $1 diff --git a/garden-service/test/integ/src/plugins/container/helpers.ts b/garden-service/test/integ/src/plugins/container/helpers.ts index 9073e951e6e..2601281f03d 100644 --- a/garden-service/test/integ/src/plugins/container/helpers.ts +++ b/garden-service/test/integ/src/plugins/container/helpers.ts @@ -6,10 +6,8 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import td from "testdouble" import { expect } from "chai" -import { containerHelpers as helpers, dockerBin } from "../../../../../src/plugins/container/helpers" -import { getLogger } from "../../../../../src/logger/logger" +import { containerHelpers as helpers } from "../../../../../src/plugins/container/helpers" describe("containerHelpers", () => { describe("getDockerVersion", () => { @@ -19,34 +17,4 @@ describe("containerHelpers", () => { expect(server).to.be.ok }) }) - - describe("getDockerCliPath", () => { - const orgEnv = { ...process.env } - const log = getLogger().placeholder() - - afterEach(() => { - process.env = orgEnv - }) - - it("should fetch the docker CLI if one is not installed", async () => { - process.env.PATH = "" - const cliPath = await helpers.getDockerCliPath(log) - const binPath = await dockerBin.getPath(log) - expect(cliPath).to.equal(binPath) - }) - - // Note: These assume the test environment has the docker CLI - it("should use the docker CLI on PATH if it is up-to-date", async () => { - td.replace(helpers, "getDockerVersion", async () => ({ client: "99.99", server: "99.99" })) - const cliPath = await helpers.getDockerCliPath(log) - expect(cliPath).to.equal("docker") - }) - - it("should fetch the docker CLI if an old one is currently on the PATH", async () => { - td.replace(helpers, "getDockerVersion", async () => ({ client: "17.03", server: "99.99" })) - const cliPath = await helpers.getDockerCliPath(log) - const binPath = await dockerBin.getPath(log) - expect(cliPath).to.equal(binPath) - }) - }) }) diff --git a/garden-service/test/integ/src/plugins/kubernetes/commands/pull-image.ts b/garden-service/test/integ/src/plugins/kubernetes/commands/pull-image.ts index b0b6731046f..0d111cfcef6 100644 --- a/garden-service/test/integ/src/plugins/kubernetes/commands/pull-image.ts +++ b/garden-service/test/integ/src/plugins/kubernetes/commands/pull-image.ts @@ -18,18 +18,13 @@ import { containerHelpers } from "../../../../../../src/plugins/container/helper import { expect } from "chai" import { LogEntry } from "../../../../../../src/logger/log-entry" import { grouped } from "../../../../../helpers" - -async function ensureImagePulled(module: Module, log: LogEntry) { - const imageId = await containerHelpers.getLocalImageId(module) - const imageHash = await containerHelpers.dockerCli(module.buildPath, ["images", "-q", imageId], log) - - expect(imageHash.stdout.length).to.be.greaterThan(0) -} +import { ContainerProvider } from "../../../../../../src/plugins/container/container" describe("pull-image plugin command", () => { let garden: Garden let graph: ConfigGraph let provider: KubernetesProvider + let containerProvider: ContainerProvider let ctx: PluginContext after(async () => { @@ -42,9 +37,22 @@ describe("pull-image plugin command", () => { garden = await getContainerTestGarden(environmentName) graph = await garden.getConfigGraph(garden.log) provider = await garden.resolveProvider("local-kubernetes") + containerProvider = await garden.resolveProvider("container") ctx = garden.getPluginContext(provider) } + async function ensureImagePulled(module: Module, log: LogEntry) { + const imageId = await containerHelpers.getLocalImageId(module) + const imageHash = await containerHelpers.dockerCli({ + cwd: module.buildPath, + args: ["images", "-q", imageId], + log, + containerProvider, + }) + + expect(imageHash.stdout.length).to.be.greaterThan(0) + } + grouped("cluster-docker", "remote-only").context("using an external cluster registry", () => { let module: Module diff --git a/garden-service/test/integ/src/plugins/kubernetes/container/build.ts b/garden-service/test/integ/src/plugins/kubernetes/container/build.ts index 82a6d02489a..d7614663729 100644 --- a/garden-service/test/integ/src/plugins/kubernetes/container/build.ts +++ b/garden-service/test/integ/src/plugins/kubernetes/container/build.ts @@ -24,11 +24,13 @@ import { dockerDaemonDeploymentName, dockerDaemonContainerName, } from "../../../../../../src/plugins/kubernetes/constants" +import { ContainerProvider } from "../../../../../../src/plugins/container/container" describe("kubernetes build flow", () => { let garden: Garden let graph: ConfigGraph let provider: KubernetesProvider + let containerProvider: ContainerProvider let ctx: PluginContext after(async () => { @@ -41,6 +43,7 @@ describe("kubernetes build flow", () => { garden = await getContainerTestGarden(environmentName) graph = await garden.getConfigGraph(garden.log) provider = await garden.resolveProvider("local-kubernetes") + containerProvider = await garden.resolveProvider("container") ctx = garden.getPluginContext(provider) } @@ -78,7 +81,12 @@ describe("kubernetes build flow", () => { const remoteId = await containerHelpers.getDeploymentImageId(module, provider.config.deploymentRegistry) // This throws if the image doesn't exist - await containerHelpers.dockerCli(module.buildPath, ["manifest", "inspect", remoteId], garden.log) + await containerHelpers.dockerCli({ + cwd: module.buildPath, + args: ["manifest", "inspect", remoteId], + log: garden.log, + containerProvider, + }) }) it("should get the build status from the deploymentRegistry", async () => { @@ -92,7 +100,12 @@ describe("kubernetes build flow", () => { }) const remoteId = await containerHelpers.getDeploymentImageId(module, provider.config.deploymentRegistry) - await containerHelpers.dockerCli(module.buildPath, ["rmi", remoteId], garden.log) + await containerHelpers.dockerCli({ + cwd: module.buildPath, + args: ["rmi", remoteId], + log: garden.log, + containerProvider, + }) const status = await k8sGetContainerBuildStatus({ ctx, diff --git a/garden-service/test/unit/src/actions.ts b/garden-service/test/unit/src/actions.ts index db5fc33efd5..5e625a5d8b8 100644 --- a/garden-service/test/unit/src/actions.ts +++ b/garden-service/test/unit/src/actions.ts @@ -104,7 +104,8 @@ describe("ActionRouter", () => { configStore: garden.configStore, projectName: garden.projectName, projectRoot: garden.projectRoot, - dependencies: [], + dependencies: {}, + tools: {}, }) expect(result).to.eql({ config, diff --git a/garden-service/test/unit/src/commands/plugins.ts b/garden-service/test/unit/src/commands/plugins.ts index b4063409f40..d66687a0b85 100644 --- a/garden-service/test/unit/src/commands/plugins.ts +++ b/garden-service/test/unit/src/commands/plugins.ts @@ -96,7 +96,7 @@ describe("PluginsCommand", () => { expect(infoLog).to.equal(dedent` USAGE - garden [global options] [args ...] + garden [global options] -- [args ...] PLUGIN COMMANDS test-plugin-a command-a Description for command A diff --git a/garden-service/test/unit/src/commands/tools.ts b/garden-service/test/unit/src/commands/tools.ts new file mode 100644 index 00000000000..c242c9c14ef --- /dev/null +++ b/garden-service/test/unit/src/commands/tools.ts @@ -0,0 +1,290 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { exec, getPlatform, getArchitecture } from "../../../../src/util/util" +import { + makeTempDir, + TempDirectory, + TestGarden, + withDefaultGlobalOpts, + dataDir, + getLogMessages, + expectError, +} from "../../../helpers" +import { expect } from "chai" +import { DEFAULT_API_VERSION } from "../../../../src/constants" +import { createGardenPlugin } from "../../../../src/types/plugin/plugin" +import { pick } from "lodash" +import { join } from "path" +import { ToolsCommand } from "../../../../src/commands/tools" +import { LogLevel } from "../../../../src/logger/log-node" +import { dedent } from "../../../../src/util/string" +import { LogEntry } from "../../../../src/logger/log-entry" +import { makeDummyGarden } from "../../../../src/cli/cli" + +describe("ToolsCommand", () => { + let tmpDir: TempDirectory + let garden: TestGarden + let log: LogEntry + + const pluginA = createGardenPlugin({ + name: "test-a", + dependencies: [], + tools: [ + { + name: "tool", + description: "foo", + type: "binary", + builds: [ + { + platform: getPlatform(), + architecture: getArchitecture(), + url: "file://" + join(dataDir, "tools", "tool-a.sh"), + sha256: "90b5248d2fc6106bdf3e5a66e8efd54383b6c4258725e9d455efb7ee32a64223", + }, + ], + }, + { + name: "lib", + description: "foo", + type: "library", + builds: [ + { + platform: getPlatform(), + architecture: getArchitecture(), + url: "file://" + join(dataDir, "tools", "tool-a.sh"), + sha256: "90b5248d2fc6106bdf3e5a66e8efd54383b6c4258725e9d455efb7ee32a64223", + }, + ], + }, + ], + }) + + const pluginB = createGardenPlugin({ + name: "test-b", + dependencies: [], + tools: [ + { + name: "tool", + description: "foo", + type: "binary", + builds: [ + { + platform: getPlatform(), + architecture: getArchitecture(), + url: "file://" + join(dataDir, "tools", "tool-b.sh"), + sha256: "b770f87151d8be76214960ecaa45de1b4a892930f1989f28de02bc2f44047ef5", + }, + ], + }, + ], + }) + + const command = new ToolsCommand() + + before(async () => { + tmpDir = await makeTempDir() + await exec("git", ["init"], { cwd: tmpDir.path }) + + garden = await TestGarden.factory(tmpDir.path, { + plugins: [pluginA, pluginB], + config: { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "test", + path: tmpDir.path, + defaultEnvironment: "default", + dotIgnoreFiles: [], + environments: [{ name: "default", variables: {} }], + providers: [{ name: "test-a" }], + variables: {}, + }, + }) + log = garden.log + + const _garden = garden as any + + _garden.providerConfigs = [{ name: "test-a" }] + _garden.registeredPlugins = pick(garden["registeredPlugins"], ["test-a", "test-b"]) + }) + + it("should list tools with no name specified", async () => { + const result = await command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: { tool: undefined }, + opts: withDefaultGlobalOpts({ "get-path": false }), + }) + + const infoLog = getLogMessages(log, (entry) => entry.level === LogLevel.info) + .join("\n") + .trim() + .split("\n") + .map((line) => line.trimEnd()) + .join("\n") + + expect(infoLog).to.equal(dedent` + USAGE + + garden [global options] -- [args ...] + garden [global options] --get-path + + PLUGIN TOOLS + test-a.tool [binary] foo + test-a.lib [library] foo + test-b.tool [binary] foo + `) + + expect(result).to.eql({}) + }) + + it("should run a configured provider's tool when using name only", async () => { + const result: any = await command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: { tool: "tool", _: ["0"] }, + opts: withDefaultGlobalOpts({ "get-path": false, "output": "json" }), + }) + + expect(result.exitCode).to.equal(0) + expect(result.stdout).to.equal("test-a") + expect(result.stderr).to.equal("") + }) + + it("should throw on an invalid tool name", async () => { + await expectError( + () => + command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: { tool: "51616ok3xnnz....361.2362&123", _: ["0"] }, + opts: withDefaultGlobalOpts({ "get-path": false, "output": "json" }), + }), + (err) => + expect(err.message).to.equal( + "Invalid tool name argument. Please specify either a tool name (no periods) or .." + ) + ) + }) + + it("should throw when plugin name is not found", async () => { + await expectError( + () => + command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: { tool: "bla.tool", _: ["0"] }, + opts: withDefaultGlobalOpts({ "get-path": false, "output": "json" }), + }), + (err) => expect(err.message).to.equal("Could not find plugin bla.") + ) + }) + + it("should throw when tool name is not found", async () => { + await expectError( + () => + command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: { tool: "bla", _: ["0"] }, + opts: withDefaultGlobalOpts({ "get-path": false, "output": "json" }), + }), + (err) => expect(err.message).to.equal("Could not find tool bla.") + ) + }) + + it("should run a tool by name when run outside of a project", async () => { + const _garden: any = await makeDummyGarden(tmpDir.path, { noPlatform: true }) + _garden.registeredPlugins = pick(garden["registeredPlugins"], ["test-a", "test-b"]) + + const result: any = await command.action({ + garden: _garden, + log, + headerLog: log, + footerLog: log, + args: { tool: "tool", _: ["0"] }, + opts: withDefaultGlobalOpts({ "get-path": false, "output": "json" }), + }) + + expect(result.exitCode).to.equal(0) + expect(result.stdout).to.equal("test-a") + expect(result.stderr).to.equal("") + }) + + it("should run a tool by plugin name and tool name", async () => { + const result: any = await command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: { tool: "test-b.tool", _: ["0"] }, + opts: withDefaultGlobalOpts({ "get-path": false, "output": "json" }), + }) + + expect(result.exitCode).to.equal(0) + expect(result.stdout).to.equal("test-b") + expect(result.stderr).to.equal("") + }) + + it("should show the path of a library", async () => { + const result: any = await command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: { tool: "test-a.lib" }, + opts: withDefaultGlobalOpts({ "get-path": false, "output": "json" }), + }) + + expect(result.path.endsWith("tool-a.sh")).to.be.true + expect(result.exitCode).to.not.exist + expect(result.stdout).to.not.exist + expect(result.stderr).to.not.exist + }) + + it("should show the path of a binary with --get-path", async () => { + const result: any = await command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: { tool: "test-a.tool" }, + opts: withDefaultGlobalOpts({ "get-path": true, "output": "json" }), + }) + + expect(result.path.endsWith("tool-a.sh")).to.be.true + expect(result.exitCode).to.not.exist + expect(result.stdout).to.not.exist + expect(result.stderr).to.not.exist + }) + + it("should return the exit code from a command", async () => { + const result: any = await command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: { tool: "tool", _: ["1"] }, + opts: withDefaultGlobalOpts({ "get-path": false, "output": "json" }), + }) + + expect(result.exitCode).to.equal(1) + expect(result.stdout).to.equal("test-a") + expect(result.stderr).to.equal("") + }) +}) diff --git a/garden-service/test/unit/src/commands/util.ts b/garden-service/test/unit/src/commands/util.ts new file mode 100644 index 00000000000..caab61a1eaa --- /dev/null +++ b/garden-service/test/unit/src/commands/util.ts @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { exec, getPlatform, getArchitecture } from "../../../../src/util/util" +import { makeTempDir, TempDirectory, TestGarden, withDefaultGlobalOpts } from "../../../helpers" +import { FetchToolsCommand } from "../../../../src/commands/util" +import { expect } from "chai" +import { DEFAULT_API_VERSION } from "../../../../src/constants" +import { createGardenPlugin } from "../../../../src/types/plugin/plugin" +import { pick } from "lodash" +import { homedir } from "os" +import { join } from "path" + +describe("FetchToolsCommand", () => { + let tmpDir: TempDirectory + + const plugin = createGardenPlugin({ + name: "test", + dependencies: [], + tools: [ + { + name: "tool", + description: "foo", + type: "binary", + builds: [ + { + platform: getPlatform(), + architecture: getArchitecture(), + url: "https://raw.githubusercontent.com/garden-io/garden/v0.11.14/.editorconfig", + sha256: "11f041ba6de46f9f4816afce861f0832e12ede015933f3580d0f6322d3906972", + }, + ], + }, + ], + }) + + const expectedPath = join(homedir(), "tools", "tool", "058921ab05f721bb", ".editorconfig") + + before(async () => { + tmpDir = await makeTempDir() + await exec("git", ["init"], { cwd: tmpDir.path }) + }) + + it("should fetch tools for configured providers", async () => { + const garden: any = await TestGarden.factory(tmpDir.path, { + plugins: [plugin], + config: { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "test", + path: tmpDir.path, + defaultEnvironment: "default", + dotIgnoreFiles: [], + environments: [{ name: "default", variables: {} }], + providers: [{ name: "test" }], + variables: {}, + }, + }) + + garden.providerConfigs = [{ name: "test" }] + garden.registeredPlugins = pick(garden["registeredPlugins"], "test") + + await garden.resolveProviders() + + const log = garden.log + const command = new FetchToolsCommand() + + const result = await command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: {}, + opts: withDefaultGlobalOpts({ all: false }), + }) + + expect(result).to.eql({ + result: { + "test.tool": { + type: "binary", + path: expectedPath, + }, + }, + }) + }) + + it("should fetch tools for all configured providers with --all", async () => { + const garden: any = await TestGarden.factory(tmpDir.path, { + plugins: [plugin], + config: { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "test", + path: tmpDir.path, + defaultEnvironment: "default", + dotIgnoreFiles: [], + environments: [{ name: "default", variables: {} }], + providers: [], + variables: {}, + }, + }) + + garden.providerConfigs = [] + garden.registeredPlugins = pick(garden["registeredPlugins"], "test") + + await garden.resolveProviders() + + const log = garden.log + const command = new FetchToolsCommand() + + const result = await command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: {}, + opts: withDefaultGlobalOpts({ all: true }), + }) + + expect(result).to.eql({ + result: { + "test.tool": { + type: "binary", + path: expectedPath, + }, + }, + }) + }) +}) diff --git a/garden-service/test/unit/src/config/config-context.ts b/garden-service/test/unit/src/config/config-context.ts index b89edc8369b..132c68c0f8b 100644 --- a/garden-service/test/unit/src/config/config-context.ts +++ b/garden-service/test/unit/src/config/config-context.ts @@ -23,7 +23,7 @@ import { prepareRuntimeContext } from "../../../../src/runtime-context" import { Service } from "../../../../src/types/service" import stripAnsi = require("strip-ansi") import { resolveTemplateString } from "../../../../src/template-string" -import { fromPairs } from "lodash" +import { fromPairs, keyBy } from "lodash" type TestValue = string | ConfigContext | TestValues | TestValueFunction type TestValueFunction = () => TestValue | Promise @@ -358,7 +358,7 @@ describe("ModuleConfigContext", () => { c = new ModuleConfigContext({ garden, - resolvedProviders: await garden.resolveProviders(), + resolvedProviders: keyBy(await garden.resolveProviders(), "name"), variables: garden.variables, secrets: { someSecret: "someSecretValue" }, dependencyConfigs: modules, @@ -500,7 +500,7 @@ describe("ModuleConfigContext", () => { withRuntime = new ModuleConfigContext({ garden, - resolvedProviders: await garden.resolveProviders(), + resolvedProviders: keyBy(await garden.resolveProviders(), "name"), variables: garden.variables, secrets, dependencyConfigs: modules, diff --git a/garden-service/test/unit/src/config/workflow.ts b/garden-service/test/unit/src/config/workflow.ts index 3aed94e7e57..99ec63be72d 100644 --- a/garden-service/test/unit/src/config/workflow.ts +++ b/garden-service/test/unit/src/config/workflow.ts @@ -10,12 +10,12 @@ import { expect } from "chai" import { DEFAULT_API_VERSION } from "../../../../src/constants" import { expectError, makeTestGardenA, TestGarden } from "../../../helpers" import { WorkflowConfig, resolveWorkflowConfig } from "../../../../src/config/workflow" -import { Provider } from "../../../../src/config/provider" +import { ProviderMap } from "../../../../src/config/provider" import { defaultContainerLimits } from "../../../../src/plugins/container/config" describe("resolveWorkflowConfig", () => { let garden: TestGarden - let resolvedProviders: Provider[] + let resolvedProviders: ProviderMap const defaults = { limits: defaultContainerLimits, keepAliveHours: 48, diff --git a/garden-service/test/unit/src/garden.ts b/garden-service/test/unit/src/garden.ts index e487308292b..1eb7f78dedb 100644 --- a/garden-service/test/unit/src/garden.ts +++ b/garden-service/test/unit/src/garden.ts @@ -37,7 +37,7 @@ import { ProjectConfig } from "../../../src/config/project" import { ModuleConfig, baseModuleSpecSchema, baseBuildSpecSchema } from "../../../src/config/module" import { DEFAULT_API_VERSION } from "../../../src/constants" import { providerConfigBaseSchema } from "../../../src/config/provider" -import { keyBy, set } from "lodash" +import { keyBy, set, mapValues } from "lodash" import stripAnsi from "strip-ansi" import { joi } from "../../../src/config/common" import { defaultDotIgnoreFiles } from "../../../src/util/fs" @@ -117,7 +117,7 @@ describe("Garden", () => { environments: ["local"], path: projectRoot, }, - dependencies: [], + dependencies: {}, moduleConfigs: [], status: { ready: true, @@ -127,25 +127,19 @@ describe("Garden", () => { expect(garden.projectName).to.equal("test-project-a") - expect(await garden.resolveProviders()).to.eql([ - emptyProvider(projectRoot, "exec"), - emptyProvider(projectRoot, "container"), - testPluginProvider, - { + const providers = await garden.resolveProviders() + const configs = mapValues(providers, (p) => p.config) + + expect(configs).to.eql({ + "exec": emptyProvider(projectRoot, "exec").config, + "container": emptyProvider(projectRoot, "container").config, + "test-plugin": testPluginProvider.config, + "test-plugin-b": { name: "test-plugin-b", - config: { - name: "test-plugin-b", - environments: ["local"], - path: projectRoot, - }, - dependencies: [testPluginProvider], - moduleConfigs: [], - status: { - ready: true, - outputs: {}, - }, + environments: ["local"], + path: projectRoot, }, - ]) + }) expect(garden.variables).to.eql({ some: "variable", @@ -163,23 +157,17 @@ describe("Garden", () => { delete process.env.TEST_PROVIDER_TYPE delete process.env.TEST_VARIABLE - expect(await garden.resolveProviders()).to.eql([ - emptyProvider(projectRoot, "exec"), - emptyProvider(projectRoot, "container"), - { + const providers = await garden.resolveProviders() + const configs = mapValues(providers, (p) => p.config) + + expect(configs).to.eql({ + "exec": emptyProvider(projectRoot, "exec").config, + "container": emptyProvider(projectRoot, "container").config, + "test-plugin": { name: "test-plugin", - config: { - name: "test-plugin", - path: projectRoot, - }, - dependencies: [], - moduleConfigs: [], - status: { - ready: true, - outputs: {}, - }, + path: projectRoot, }, - ]) + }) expect(garden.variables).to.eql({ "some": "banana", @@ -768,6 +756,60 @@ describe("Garden", () => { expect(findByName(parsed.commands!, "bar")).to.eql(foo.commands[1]) }) + it("should combine tools from both plugins, ignoring base tools when overriding", async () => { + const base = createGardenPlugin({ + name: "base", + tools: [ + { + name: "base-tool", + type: "binary", + description: "Test", + builds: [], + }, + { + name: "common-tool", + type: "binary", + description: "Base description", + builds: [], + }, + ], + }) + const foo = createGardenPlugin({ + name: "foo", + base: "base", + tools: [ + { + name: "common-tool", + type: "library", + description: "Different description", + builds: [], + }, + { + name: "different-tool", + type: "binary", + description: "Test", + builds: [], + }, + ], + }) + + const garden = await TestGarden.factory(pathFoo, { + plugins: [base, foo], + config: projectConfigFoo, + }) + + const parsed = await garden.getPlugin("foo") + + expect(parsed.tools!.length).to.equal(3) + expect(findByName(parsed.tools!, "base-tool")).to.eql({ + ...base.tools![0], + }) + expect(findByName(parsed.tools!, "common-tool")).to.eql({ + ...foo.tools![0], + }) + expect(findByName(parsed.tools!, "different-tool")).to.eql(foo.tools![1]) + }) + it("should register module types from both plugins", async () => { const base = createGardenPlugin({ name: "base", @@ -1360,25 +1402,19 @@ describe("Garden", () => { }, } - expect(await garden.resolveProviders()).to.eql([ - emptyProvider(projectRoot, "exec"), - emptyProvider(projectRoot, "container"), - testPluginProvider, - { + const providers = await garden.resolveProviders() + const configs = mapValues(providers, (p) => p.config) + + expect(configs).to.eql({ + "exec": emptyProvider(projectRoot, "exec").config, + "container": emptyProvider(projectRoot, "container").config, + "test-plugin": testPluginProvider.config, + "test-plugin-b": { name: "test-plugin-b", - config: { - name: "test-plugin-b", - environments: ["local"], - path: projectRoot, - }, - dependencies: [testPluginProvider], - moduleConfigs: [], - status: { - ready: true, - outputs: {}, - }, + environments: ["local"], + path: projectRoot, }, - ]) + }) }) it("should call a configureProvider handler if applicable", async () => { @@ -1923,7 +1959,7 @@ describe("Garden", () => { const providerA = await garden.resolveProvider("test-a") const providerB = await garden.resolveProvider("test-b") - expect(providerB.dependencies).to.eql([providerA]) + expect(providerB.dependencies).to.eql({ "test-a": providerA }) }) it("should match a dependency to a plugin base that's declared by multiple plugins", async () => { @@ -1974,7 +2010,7 @@ describe("Garden", () => { const providerB = await garden.resolveProvider("test-b") const providerC = await garden.resolveProvider("test-c") - expect(providerC.dependencies).to.eql([providerA, providerB]) + expect(providerC.dependencies).to.eql({ "test-a": providerA, "test-b": providerB }) }) context("when a plugin has a base", () => { @@ -3459,7 +3495,7 @@ function emptyProvider(projectRoot: string, name: string) { name, path: projectRoot, }, - dependencies: [], + dependencies: {}, moduleConfigs: [], status: { ready: true, diff --git a/garden-service/test/unit/src/plugins/container/container.ts b/garden-service/test/unit/src/plugins/container/container.ts index eddd8c2e0e7..432ecea2b7a 100644 --- a/garden-service/test/unit/src/plugins/container/container.ts +++ b/garden-service/test/unit/src/plugins/container/container.ts @@ -13,7 +13,7 @@ import td from "testdouble" import { Garden } from "../../../../../src/garden" import { PluginContext } from "../../../../../src/plugin-context" -import { gardenPlugin } from "../../../../../src/plugins/container/container" +import { gardenPlugin, ContainerProvider } from "../../../../../src/plugins/container/container" import { dataDir, expectError, makeTestGarden } from "../../../../helpers" import { moduleFromConfig } from "../../../../../src/types/module" import { ModuleConfig } from "../../../../../src/config/module" @@ -73,12 +73,13 @@ describe("plugins.container", () => { let garden: Garden let ctx: PluginContext let log: LogEntry + let containerProvider: ContainerProvider beforeEach(async () => { garden = await makeTestGarden(projectRoot, { plugins: [gardenPlugin] }) log = garden.log - const provider = await garden.resolveProvider("container") - ctx = garden.getPluginContext(provider) + containerProvider = await garden.resolveProvider("container") + ctx = garden.getPluginContext(containerProvider) td.replace(garden.buildDir, "syncDependencyProducts", () => null) @@ -773,9 +774,10 @@ describe("plugins.container", () => { const cmdArgs = ["build", "-t", "some/image", module.buildPath] - td.replace(helpers, "dockerCli", async (path: string, args: string[]) => { - expect(path).to.equal(module.buildPath) + td.replace(helpers, "dockerCli", async ({ cwd, args, containerProvider: provider }) => { + expect(cwd).to.equal(module.buildPath) expect(args).to.eql(cmdArgs) + expect(provider).to.exist return { all: "log" } }) @@ -800,9 +802,10 @@ describe("plugins.container", () => { const cmdArgs = ["build", "-t", "some/image", "--target", "foo", module.buildPath] - td.replace(helpers, "dockerCli", async (path: string, args: string[]) => { - expect(path).to.equal(module.buildPath) + td.replace(helpers, "dockerCli", async ({ cwd, args, containerProvider: provider }) => { + expect(cwd).to.equal(module.buildPath) expect(args).to.eql(cmdArgs) + expect(provider).to.exist return { all: "log" } }) @@ -835,9 +838,10 @@ describe("plugins.container", () => { module.buildPath, ] - td.replace(helpers, "dockerCli", async (path: string, args: string[]) => { - expect(path).to.equal(module.buildPath) + td.replace(helpers, "dockerCli", async ({ cwd, args, containerProvider: provider }) => { + expect(cwd).to.equal(module.buildPath) expect(args).to.eql(cmdArgs) + expect(provider).to.exist return { all: "log" } }) @@ -872,16 +876,15 @@ describe("plugins.container", () => { td.replace(helpers, "getLocalImageId", async () => "some/image:12345") td.replace(helpers, "getPublicImageId", async () => "some/image:12345") - const dockerCli = td.replace(helpers, "dockerCli") + td.replace(helpers, "dockerCli", async ({ cwd, args, containerProvider: provider }) => { + expect(cwd).to.equal(module.buildPath) + expect(args).to.eql(["push", "some/image:12345"]) + expect(provider).to.exist + return { all: "log" } + }) const result = await publishModule({ ctx, log, module }) expect(result).to.eql({ message: "Published some/image:12345", published: true }) - - td.verify(dockerCli(module.buildPath, ["tag", "some/image:12345", "some/image:12345"]), { - ignoreExtraArgs: true, - times: 0, - }) - td.verify(dockerCli(module.buildPath, ["push", "some/image:12345"]), { ignoreExtraArgs: true }) }) it("should tag image if remote id differs from local id", async () => { @@ -898,50 +901,22 @@ describe("plugins.container", () => { const result = await publishModule({ ctx, log, module }) expect(result).to.eql({ message: "Published some/image:1.1", published: true }) - td.verify(dockerCli(module.buildPath, ["tag", "some/image:12345", "some/image:1.1"]), { ignoreExtraArgs: true }) - td.verify(dockerCli(module.buildPath, ["push", "some/image:1.1"]), { ignoreExtraArgs: true }) - }) - }) - - describe("checkDockerClientVersion", () => { - it("should return if client version is equal to the minimum version", async () => { - helpers.checkDockerClientVersion(minDockerVersion) - }) - - it("should return if client version is greater than the minimum version", async () => { - const version = { - client: "99.99", - server: "99.99", - } - - helpers.checkDockerClientVersion(version) - }) - - it("should throw if client is not installed (version is undefined)", async () => { - const version = { - client: undefined, - server: minDockerVersion.server, - } - - await expectError( - () => helpers.checkDockerClientVersion(version), - (err) => { - expect(err.message).to.equal("Docker client is not installed.") - } + td.verify( + dockerCli({ + cwd: module.buildPath, + args: ["tag", "some/image:12345", "some/image:1.1"], + log: td.matchers.anything(), + containerProvider: td.matchers.anything(), + }) ) - }) - - it("should throw if client version is too old", async () => { - const version = { - client: "17.06", - server: minDockerVersion.server, - } - await expectError( - () => helpers.checkDockerClientVersion(version), - (err) => { - expect(err.message).to.equal("Docker client needs to be version 19.03.0 or newer (got 17.06)") - } + td.verify( + dockerCli({ + cwd: module.buildPath, + args: ["push", "some/image:1.1"], + log: td.matchers.anything(), + containerProvider: td.matchers.anything(), + }) ) }) }) diff --git a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts index da5f07291e8..394ae03e8fa 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts @@ -30,6 +30,9 @@ import { ContainerServiceSpec, } from "../../../../../../src/plugins/container/config" import { defaultSystemNamespace } from "../../../../../../src/plugins/kubernetes/system" +import { PluginTools } from "../../../../../../src/types/plugin/tools" +import { keyBy } from "lodash" +import { PluginTool } from "../../../../../../src/util/ext-tools" const kubeConfigEnvVar = process.env.KUBECONFIG const namespace = "my-namespace" @@ -66,14 +69,6 @@ const basicConfig: KubernetesConfig = { _systemServices: [], } -const basicProvider: KubernetesProvider = { - name: "kubernetes", - config: basicConfig, - dependencies: [], - moduleConfigs: [], - status: { ready: true, outputs: {} }, -} - const singleTlsConfig: KubernetesConfig = { ...basicConfig, forceSsl: true, @@ -88,14 +83,6 @@ const singleTlsConfig: KubernetesConfig = { ], } -const singleTlsProvider: KubernetesProvider = { - name: "kubernetes", - config: singleTlsConfig, - dependencies: [], - moduleConfigs: [], - status: { ready: true, outputs: {} }, -} - const multiTlsConfig: KubernetesConfig = { ...basicConfig, forceSsl: true, @@ -124,14 +111,6 @@ const multiTlsConfig: KubernetesConfig = { ], } -const multiTlsProvider: KubernetesProvider = { - name: "kubernetes", - config: multiTlsConfig, - dependencies: [], - moduleConfigs: [], - status: { ready: true, outputs: {} }, -} - // generated with `openssl req -newkey rsa:2048 -nodes -keyout key.pem -x509 -days 365 -out certificate.pem` const myDomainCrt = `-----BEGIN CERTIFICATE----- MIIDgDCCAmgCCQCf3b7n4GtdljANBgkqhkiG9w0BAQsFADCBgTELMAkGA1UEBhMC @@ -328,6 +307,10 @@ describe("createIngressResources", () => { const configure = plugin.createModuleTypes![0].handlers.configure! let garden: Garden + let tools: PluginTools + let basicProvider: KubernetesProvider + let singleTlsProvider: KubernetesProvider + let multiTlsProvider: KubernetesProvider before(() => { process.env.KUBECONFIG = join(projectRoot, "kubeconfig.yml") @@ -343,6 +326,11 @@ describe("createIngressResources", () => { beforeEach(async () => { garden = await makeTestGarden(projectRoot, { plugins: [gardenPlugin] }) + const k8sPlugin = garden.registeredPlugins.kubernetes + tools = keyBy( + (k8sPlugin.tools || []).map((t) => new PluginTool(t)), + "name" + ) td.replace(garden.buildDir, "syncDependencyProducts", () => null) @@ -351,6 +339,33 @@ describe("createIngressResources", () => { dependencyVersions: {}, files: [], })) + + basicProvider = { + name: "kubernetes", + config: basicConfig, + dependencies: {}, + moduleConfigs: [], + status: { ready: true, outputs: {} }, + tools, + } + + multiTlsProvider = { + name: "kubernetes", + config: multiTlsConfig, + dependencies: {}, + moduleConfigs: [], + status: { ready: true, outputs: {} }, + tools, + } + + singleTlsProvider = { + name: "kubernetes", + config: singleTlsConfig, + dependencies: {}, + moduleConfigs: [], + status: { ready: true, outputs: {} }, + tools, + } }) async function getTestService(...ingresses: ContainerIngressSpec[]): Promise { @@ -670,9 +685,10 @@ describe("createIngressResources", () => { }, ], }, - dependencies: [], + dependencies: {}, moduleConfigs: [], status: { ready: true, outputs: {} }, + tools, } const err: any = new Error("nope") @@ -703,9 +719,10 @@ describe("createIngressResources", () => { }, ], }, - dependencies: [], + dependencies: {}, moduleConfigs: [], status: { ready: true, outputs: {} }, + tools, } const api = await getKubeApi(basicProvider) @@ -738,9 +755,10 @@ describe("createIngressResources", () => { }, ], }, - dependencies: [], + dependencies: {}, moduleConfigs: [], status: { ready: true, outputs: {} }, + tools, } const api = await getKubeApi(basicProvider) @@ -833,9 +851,10 @@ describe("createIngressResources", () => { }, ], }, - dependencies: [], + dependencies: {}, moduleConfigs: [], status: { ready: true, outputs: {} }, + tools, } td.when(api.core.readNamespacedSecret("foo", "default")).thenResolve(myDomainCertSecret) diff --git a/garden-service/test/unit/src/plugins/kubernetes/init.ts b/garden-service/test/unit/src/plugins/kubernetes/init.ts index 9913b4dc884..c3023a33f8e 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/init.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/init.ts @@ -62,9 +62,10 @@ const basicConfig: KubernetesConfig = { const basicProvider: KubernetesProvider = { name: "kubernetes", config: basicConfig, - dependencies: [], + dependencies: {}, moduleConfigs: [], status: { ready: true, outputs: {} }, + tools: {}, } const dockerSimpleAuthSecret: KubernetesResource = { diff --git a/garden-service/test/unit/src/plugins/kubernetes/kubernetes.ts b/garden-service/test/unit/src/plugins/kubernetes/kubernetes.ts index 9c6b606ef91..2507d134236 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/kubernetes.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/kubernetes.ts @@ -12,6 +12,8 @@ import { defaultSystemNamespace } from "../../../../../src/plugins/kubernetes/sy import { makeDummyGarden } from "../../../../../src/cli/cli" import { expect } from "chai" import { TempDirectory, makeTempDir, grouped } from "../../../../helpers" +import { keyBy } from "lodash" +import { PluginTool } from "../../../../../src/util/ext-tools" describe("kubernetes configureProvider", () => { const basicConfig: KubernetesConfig = { @@ -50,6 +52,7 @@ describe("kubernetes configureProvider", () => { ...basicConfig, buildMode: "cluster-docker", } + const plugin = garden.registeredPlugins.kubernetes const result = await configureProvider({ environmentName: "default", @@ -57,8 +60,12 @@ describe("kubernetes configureProvider", () => { projectRoot: garden.projectRoot, config, log: garden.log, - dependencies: [], + dependencies: {}, configStore: garden.configStore, + tools: keyBy( + plugin.tools?.map((t) => new PluginTool(t)), + "name" + ), }) expect(result.config.deploymentRegistry).to.eql({ @@ -77,6 +84,7 @@ describe("kubernetes configureProvider", () => { namespace: "my-namespace", }, } + const plugin = garden.registeredPlugins.kubernetes const result = await configureProvider({ environmentName: "default", @@ -84,8 +92,12 @@ describe("kubernetes configureProvider", () => { projectRoot: garden.projectRoot, config, log: garden.log, - dependencies: [], + dependencies: {}, configStore: garden.configStore, + tools: keyBy( + plugin.tools?.map((t) => new PluginTool(t)), + "name" + ), }) expect(result.config.deploymentRegistry).to.eql({