From 99b5c7200a9dda3cd021bd9be133d50011857907 Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Mon, 22 Feb 2021 22:37:11 +0100 Subject: [PATCH] feat(config): allow sparse arrays where appropriate in config schemas Users can now place undefined or null values in place of real values in many array configuration values, in order to conditionally place values there. Generally, this applies for any array schema that wouldn't naturally expect null values (which would make the intent ambiguous). For example, you might do something like `dependencies: ["service-a", ${var.some-condition ? 'service-b' : null}]` to conditionally include a service dependency. --- core/src/config/common.ts | 17 ++++++++++++++ core/src/config/module.ts | 7 +++--- core/src/config/project.ts | 13 ++++++----- core/src/config/provider.ts | 4 ++-- core/src/config/service.ts | 4 ++-- core/src/config/task.ts | 4 ++-- core/src/config/test.ts | 4 ++-- core/src/config/workflow.ts | 14 +++++++++--- core/src/docs/common.ts | 4 ++++ core/src/docs/config.ts | 5 +++-- core/src/docs/joi-schema.ts | 8 +++---- core/src/plugins/container/config.ts | 14 ++++++------ core/src/plugins/exec.ts | 8 +++---- core/src/plugins/kubernetes/config.ts | 11 +++++----- core/src/plugins/kubernetes/helm/config.ts | 10 ++++----- .../kubernetes/kubernetes-module/config.ts | 16 +++++++------- .../volumes/persistentvolumeclaim.ts | 4 ++-- .../maven-container/maven-container.ts | 4 ++-- core/src/plugins/openfaas/config.ts | 6 ++--- core/test/unit/src/actions.ts | 1 + core/test/unit/src/config/common.ts | 15 +++++++++++++ core/test/unit/src/config/service.ts | 22 +++++++++++++++++++ core/test/unit/src/config/workflow.ts | 1 + core/test/unit/src/plugins/exec.ts | 6 +++++ docs/reference/module-template-config.md | 6 ++--- docs/reference/module-types/conftest.md | 6 ++--- docs/reference/module-types/container.md | 6 ++--- docs/reference/module-types/exec.md | 18 +++++++-------- docs/reference/module-types/hadolint.md | 6 ++--- docs/reference/module-types/helm.md | 6 ++--- docs/reference/module-types/kubernetes.md | 6 ++--- .../reference/module-types/maven-container.md | 6 ++--- docs/reference/module-types/openfaas.md | 6 ++--- .../module-types/persistentvolumeclaim.md | 6 ++--- docs/reference/module-types/templated.md | 6 ++--- docs/reference/module-types/terraform.md | 6 ++--- docs/reference/workflow-config.md | 12 +++++----- plugins/conftest/index.ts | 4 ++-- yarn.lock | 18 +++++++-------- 39 files changed, 199 insertions(+), 121 deletions(-) create mode 100644 core/test/unit/src/config/service.ts diff --git a/core/src/config/common.ts b/core/src/config/common.ts index 796e687658..dd35f1907f 100644 --- a/core/src/config/common.ts +++ b/core/src/config/common.ts @@ -119,6 +119,7 @@ export interface Schema extends Joi.Root { gitUrl: () => GitUrlSchema posixPath: () => PosixPathSchema hostname: () => Joi.StringSchema + sparseArray: () => Joi.ArraySchema } export let joi: Schema = Joi.extend({ @@ -393,6 +394,19 @@ joi = joi.extend({ }, }) +/** + * Add a joi.sparseArray() type, that both allows sparse arrays _and_ filters the falsy values out. + */ +joi = joi.extend({ + base: Joi.array().sparse(true), + type: "sparseArray", + coerce: { + method(value) { + return { value: value && value.filter((v: any) => v !== undefined && v !== null) } + }, + }, +}) + export const joiPrimitive = () => joi .alternatives() @@ -482,6 +496,9 @@ export const joiEnvVars = () => export const joiArray = (schema: Joi.Schema) => joi.array().items(schema).default([]) +// This allows null, empty string or undefined values on the item values and then filters them out +export const joiSparseArray = (schema: Joi.Schema) => joi.sparseArray().items(schema.allow(null)).default([]) + export const joiRepositoryUrl = () => joi .alternatives( diff --git a/core/src/config/module.ts b/core/src/config/module.ts index 9d3b98a8c2..9d49a569e1 100644 --- a/core/src/config/module.ts +++ b/core/src/config/module.ts @@ -17,6 +17,7 @@ import { apiVersionSchema, DeepPrimitiveMap, joiVariables, + joiSparseArray, } from "./common" import { TestConfig, testConfigSchema } from "./test" import { TaskConfig, taskConfigSchema } from "./task" @@ -55,7 +56,7 @@ export const buildDependencySchema = () => joi.object().keys({ name: joi.string().required().description("Module name to build ahead of this module."), plugin: joi.string().meta({ internal: true }).description("The name of plugin that provides the build dependency."), - copy: joiArray(copySchema()).description( + copy: joiSparseArray(copySchema()).description( "Specify one or more files or directories to copy from the built dependency to this module." ), }) @@ -132,7 +133,7 @@ export const baseBuildSpecSchema = () => joi .object() .keys({ - dependencies: joiArray(buildDependencySchema()) + dependencies: joiSparseArray(buildDependencySchema()) .description("A list of modules that must be built before this module is built.") .example([{ name: "some-other-module-name" }]), }) @@ -206,7 +207,7 @@ export const baseModuleSpecKeys = () => ({ .boolean() .default(true) .description("When false, disables pushing this module to remote registries."), - generateFiles: joi.array().items(generatedFileSchema()).description(dedent` + generateFiles: joiSparseArray(generatedFileSchema()).description(dedent` A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module. `), }) diff --git a/core/src/config/project.ts b/core/src/config/project.ts index 39b785480f..9b55204fac 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -22,6 +22,7 @@ import { DeepPrimitiveMap, joiVariablesDescription, apiVersionSchema, + joiSparseArray, } from "./common" import { validateWithPath } from "./validation" import { resolveTemplateStrings } from "../template-string" @@ -134,9 +135,9 @@ export const environmentSchema = () => export const environmentsSchema = () => joi .alternatives( - joi.array().items(environmentSchema()).unique("name"), + joiSparseArray(environmentSchema()).unique("name"), // Allow a string as a shorthand for { name: foo } - joi.array().items(joiUserIdentifier()) + joiSparseArray(joiUserIdentifier()) ) .description("A list of environments to configure for the project.") @@ -158,7 +159,7 @@ export const projectSourceSchema = () => }) export const projectSourcesSchema = () => - joiArray(projectSourceSchema()).unique("name").description("A list of remote sources to import into project.") + joiSparseArray(projectSourceSchema()).unique("name").description("A list of remote sources to import into project.") export const linkedSourceSchema = () => joi.object().keys({ @@ -296,7 +297,7 @@ export const projectDocsSchema = () => .array() .items(environmentSchema()) .description((environmentsSchema().describe().flags).description), - providers: joiArray(providerConfigBaseSchema()).description( + providers: joiSparseArray(providerConfigBaseSchema()).description( "A list of providers that should be used for this project, and their configuration. " + "Please refer to individual plugins/providers for details on how to configure them." ), @@ -313,7 +314,7 @@ export const projectDocsSchema = () => ` ) .example("dev"), - dotIgnoreFiles: joiArray(joi.posixPath().filenameOnly()) + dotIgnoreFiles: joiSparseArray(joi.posixPath().filenameOnly()) .default(defaultDotIgnoreFiles) .description( deline` @@ -326,7 +327,7 @@ export const projectDocsSchema = () => ) .example([".gardenignore", ".gitignore"]), modules: projectModulesSchema().description("Control where to scan for modules in the project."), - outputs: joiArray(projectOutputSchema()) + outputs: joiSparseArray(projectOutputSchema()) .unique("name") .description( dedent` diff --git a/core/src/config/provider.ts b/core/src/config/provider.ts index b0ce186c16..db7ddebc02 100644 --- a/core/src/config/provider.ts +++ b/core/src/config/provider.ts @@ -7,7 +7,7 @@ */ import { deline } from "../util/string" -import { joiIdentifier, joiUserIdentifier, joiArray, joi, joiIdentifierMap } from "./common" +import { joiIdentifier, joiUserIdentifier, joiArray, joi, joiIdentifierMap, joiSparseArray } from "./common" import { collectTemplateReferences } from "../template-string" import { ConfigurationError } from "../exceptions" import { ModuleConfig, moduleConfigSchema } from "./module" @@ -30,7 +30,7 @@ export interface GenericProviderConfig extends BaseProviderConfig { const providerFixedFieldsSchema = () => joi.object().keys({ name: joiIdentifier().required().description("The name of the provider plugin to use.").example("local-kubernetes"), - dependencies: joiArray(joiIdentifier()) + dependencies: joiSparseArray(joiIdentifier()) .description("List other providers that should be resolved before this one.") .example(["exec"]), environments: joi diff --git a/core/src/config/service.ts b/core/src/config/service.ts index 1bfb76abb6..0e0cc02476 100644 --- a/core/src/config/service.ts +++ b/core/src/config/service.ts @@ -6,7 +6,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { joiIdentifier, joiArray, joiUserIdentifier, joi, joiVariables } from "./common" +import { joiIdentifier, joiUserIdentifier, joi, joiVariables, joiSparseArray } from "./common" import { deline, dedent } from "../util/string" /** @@ -22,7 +22,7 @@ export interface CommonServiceSpec { export const serviceOutputsSchema = joiVariables() export const dependenciesSchema = () => - joiArray(joiIdentifier()).description(deline` + joiSparseArray(joiIdentifier()).description(deline` The names of any services that this service depends on at runtime, and the names of any tasks that should be executed before this service is deployed. `) diff --git a/core/src/config/task.ts b/core/src/config/task.ts index 8f2b75693a..af315d483d 100644 --- a/core/src/config/task.ts +++ b/core/src/config/task.ts @@ -6,7 +6,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { joiArray, joiUserIdentifier, joi } from "./common" +import { joiUserIdentifier, joi, joiSparseArray } from "./common" import { deline, dedent } from "../util/string" export interface TaskSpec {} @@ -35,7 +35,7 @@ export const baseTaskSpecSchema = () => .keys({ name: joiUserIdentifier().required().description("The name of the task."), description: joi.string().optional().description("A description of the task."), - dependencies: joiArray(joi.string()).description(deline` + dependencies: joiSparseArray(joi.string()).description(deline` The names of any tasks that must be executed, and the names of any services that must be running, before this task is executed. `), disabled: joi diff --git a/core/src/config/test.ts b/core/src/config/test.ts index dccfc4b3dc..a250ffa9c2 100644 --- a/core/src/config/test.ts +++ b/core/src/config/test.ts @@ -6,7 +6,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { joiArray, joiUserIdentifier, joi } from "./common" +import { joiUserIdentifier, joi, joiSparseArray } from "./common" import { deline, dedent } from "../util/string" export interface BaseTestSpec { @@ -19,7 +19,7 @@ export interface BaseTestSpec { export const baseTestSpecSchema = () => joi.object().keys({ name: joiUserIdentifier().required().description("The name of the test."), - dependencies: joiArray(joi.string()).description(deline` + dependencies: joiSparseArray(joi.string()).description(deline` The names of any services that must be running, and the names of any tasks that must be executed, before the test is run. `), diff --git a/core/src/config/workflow.ts b/core/src/config/workflow.ts index 0f67f44447..e1a1625b74 100644 --- a/core/src/config/workflow.ts +++ b/core/src/config/workflow.ts @@ -7,7 +7,15 @@ */ import { isEqual, merge, omit, take } from "lodash" -import { joi, joiUserIdentifier, joiVariableName, joiIdentifier, joiEnvVars, PrimitiveMap } from "./common" +import { + joi, + joiUserIdentifier, + joiVariableName, + joiIdentifier, + joiEnvVars, + PrimitiveMap, + joiSparseArray, +} from "./common" import { DEFAULT_API_VERSION } from "../constants" import { deline, dedent } from "../util/string" import { defaultContainerLimits, ServiceLimitSpec } from "../plugins/container/config" @@ -64,7 +72,7 @@ export const workflowConfigSchema = () => kind: joi.string().default("Workflow").valid("Workflow"), name: joiUserIdentifier().required().description("The name of this workflow.").example("my-workflow"), description: joi.string().description("A description of the workflow."), - files: joi.array().items(workflowFileSchema()).description(dedent` + files: joiSparseArray(workflowFileSchema()).description(dedent` A list of files to write before starting the workflow. This is useful to e.g. create files required for provider authentication, and can be created from data stored in secrets or templated strings. @@ -90,7 +98,7 @@ export const workflowConfigSchema = () => .description("The maximum amount of RAM the workflow pod can use, in megabytes (i.e. 1024 = 1 GB)"), }) .default(defaultContainerLimits), - steps: joi.array().items(workflowStepSchema()).required().min(1).description(deline` + steps: joiSparseArray(workflowStepSchema()).required().min(1).description(deline` The steps the workflow should run. At least one step is required. Steps are run sequentially. If a step fails, subsequent steps are skipped. `), diff --git a/core/src/docs/common.ts b/core/src/docs/common.ts index fd946cace2..c001f92929 100644 --- a/core/src/docs/common.ts +++ b/core/src/docs/common.ts @@ -124,3 +124,7 @@ export function getGitHubUrl(path: string) { export function templateStringLiteral(key: string) { return "`${" + key + "}`" } + +export function isArrayType(type: string) { + return type === "array" || type === "sparseArray" +} diff --git a/core/src/docs/config.ts b/core/src/docs/config.ts index 2fcc5d5ed0..2464b61c88 100644 --- a/core/src/docs/config.ts +++ b/core/src/docs/config.ts @@ -22,6 +22,7 @@ import { BaseKeyDescription, NormalizeOptions, flattenSchema, + isArrayType, } from "./common" import { JoiDescription, JoiKeyDescription } from "./joi-schema" import { safeDumpYaml } from "../util/util" @@ -143,9 +144,9 @@ export function renderSchemaDescriptionYaml( const comment: string[] = [] const out: string[] = [] const isFirstChild = parent && parent === prevDesc - const isArrayItem = parent && parent.type === "array" + const isArrayItem = parent && isArrayType(parent.type) const isFirstArrayItem = isArrayItem && isFirstChild - const isPrimitive = type !== "array" && type !== "object" + const isPrimitive = !isArrayType(type) && type !== "object" const presetValue = getPresetValue(desc) diff --git a/core/src/docs/joi-schema.ts b/core/src/docs/joi-schema.ts index 04dedf24d5..a68c79db3e 100644 --- a/core/src/docs/joi-schema.ts +++ b/core/src/docs/joi-schema.ts @@ -8,7 +8,7 @@ import Joi from "@hapi/joi" import { uniq, isFunction, extend, isArray, isPlainObject } from "lodash" -import { BaseKeyDescription } from "./common" +import { BaseKeyDescription, isArrayType } from "./common" import { findByName, safeDumpYaml } from "../util/util" import { JsonKeyDescription } from "./json-schema" @@ -63,7 +63,7 @@ export class JoiKeyDescription extends BaseKeyDescription { } formatName() { - return this.type === "array" ? `${this.name}[]` : this.name + return isArrayType(this.type) ? `${this.name}[]` : this.name } formatExample() { @@ -139,7 +139,7 @@ export class JoiKeyDescription extends BaseKeyDescription { ) } return childDescriptions - } else if (this.joiDescription.type === "array" && this.joiDescription.items[0]) { + } else if (isArrayType(this.joiDescription.type) && this.joiDescription.items[0]) { // We only use the first array item return [ new JoiKeyDescription({ @@ -176,7 +176,7 @@ function getObjectSchema(d: JoiDescription) { function formatType(joiDescription: JoiDescription) { const { type } = joiDescription - const items = type === "array" && joiDescription.items + const items = isArrayType(type) && joiDescription.items if (items && items.length > 0) { // We don't consider an array of primitives as children diff --git a/core/src/plugins/container/config.ts b/core/src/plugins/container/config.ts index 27a1e985d3..aa9bd150fa 100644 --- a/core/src/plugins/container/config.ts +++ b/core/src/plugins/container/config.ts @@ -9,7 +9,6 @@ import { GardenModule, FileCopySpec } from "../../types/module" import { joiUserIdentifier, - joiArray, PrimitiveMap, joiPrimitive, joi, @@ -17,6 +16,7 @@ import { Primitive, joiModuleIncludeDirective, joiIdentifier, + joiSparseArray, } from "../../config/common" import { ArtifactSpec } from "./../../config/validation" import { Service, ingressHostnameSchema, linkUrlSchema } from "../../types/service" @@ -337,7 +337,7 @@ const volumeSchema = () => .oxor("hostPath", "module") export function getContainerVolumesSchema(targetType: string) { - return joiArray(volumeSchema()).unique("name").description(dedent` + return joiSparseArray(volumeSchema()).unique("name").description(dedent` List of volumes that should be mounted when deploying the ${targetType}. Note: If neither \`hostPath\` nor \`module\` is specified, an empty ephemeral volume is created and mounted when deploying the container. @@ -363,7 +363,7 @@ const containerServiceSchema = () => Whether to run the service as a daemon (to ensure exactly one instance runs per node). May not be supported by all providers. `), - ingresses: joiArray(ingressSchema()) + ingresses: joiSparseArray(ingressSchema()) .description("List of ingress endpoints that the service exposes.") .example([{ path: "/api", port: "http" }]), env: containerEnvVarsSchema(), @@ -387,7 +387,7 @@ const containerServiceSchema = () => ) .example(["npm", "run", "dev"]), limits: limitsSchema().description("Specify resource limits for the service.").default(defaultContainerLimits), - ports: joiArray(portSchema()).unique("name").description("List of ports that the service container exposes."), + ports: joiSparseArray(portSchema()).unique("name").description("List of ports that the service container exposes."), replicas: joi.number().integer().description(deline` The number of instances of the service to deploy. Defaults to 3 for environments configured with \`production: true\`, otherwise 1. @@ -584,12 +584,12 @@ export const containerModuleSpecSchema = () => `), hotReload: hotReloadConfigSchema(), dockerfile: joi.posixPath().subPathOnly().description("POSIX-style name of Dockerfile, relative to module root."), - services: joiArray(containerServiceSchema()) + services: joiSparseArray(containerServiceSchema()) .unique("name") .description("A list of services to deploy from this container module."), - tests: joiArray(containerTestSchema()).description("A list of tests to run in the module."), + tests: joiSparseArray(containerTestSchema()).description("A list of tests to run in the module."), // We use the user-facing term "tasks" as the key here, instead of "tasks". - tasks: joiArray(containerTaskSchema()).description(deline` + tasks: joiSparseArray(containerTaskSchema()).description(deline` A list of tasks that can be run from this container module. These can be used as dependencies for services (executed before the service is deployed) or for other tasks. `), diff --git a/core/src/plugins/exec.ts b/core/src/plugins/exec.ts index b3fc6cb1a1..ff857cab1d 100644 --- a/core/src/plugins/exec.ts +++ b/core/src/plugins/exec.ts @@ -10,7 +10,7 @@ import Bluebird from "bluebird" import { mapValues } from "lodash" import { join } from "path" import cpy = require("cpy") -import { joiArray, joiEnvVars, joi } from "../config/common" +import { joiArray, joiEnvVars, joi, joiSparseArray } from "../config/common" import { validateWithPath, ArtifactSpec } from "../config/validation" import { createGardenPlugin } from "../types/plugin/plugin" import { GardenModule, getModuleKey } from "../types/module" @@ -52,7 +52,7 @@ const artifactSchema = () => target: joi.posixPath().relativeOnly().subPathOnly().default(".").description(artifactsTargetDescription), }) -const artifactsSchema = () => joi.array().items(artifactSchema()) +const artifactsSchema = () => joiSparseArray(artifactSchema()) export interface ExecTestSpec extends BaseTestSpec { command: string[] @@ -152,8 +152,8 @@ export const execModuleSpecSchema = () => .default(false), build: execBuildSpecSchema(), env: joiEnvVars(), - tasks: joiArray(execTaskSpecSchema()).description("A list of tasks that can be run in this module."), - tests: joiArray(execTestSchema()).description("A list of tests to run in the module."), + tasks: joiSparseArray(execTaskSpecSchema()).description("A list of tasks that can be run in this module."), + tests: joiSparseArray(execTestSchema()).description("A list of tests to run in the module."), }) .unknown(false) .description("The module specification for an exec module.") diff --git a/core/src/plugins/kubernetes/config.ts b/core/src/plugins/kubernetes/config.ts index 9ccbac8db8..3e0a656ec6 100644 --- a/core/src/plugins/kubernetes/config.ts +++ b/core/src/plugins/kubernetes/config.ts @@ -16,6 +16,7 @@ import { joiStringMap, StringMap, joiIdentifierDescription, + joiSparseArray, } from "../../config/common" import { Provider, providerConfigBaseSchema, GenericProviderConfig } from "../../config/provider" import { @@ -268,7 +269,7 @@ const secretRef = joi .description("Reference to a Kubernetes secret.") const imagePullSecretsSchema = () => - joiArray(secretRef).description(dedent` + joiSparseArray(secretRef).description(dedent` References to \`docker-registry\` secrets to use for authenticating with remote registries when pulling images. This is necessary if you reference private images in your module configuration, and is required when configuring a remote Kubernetes environment with buildMode=local. @@ -488,7 +489,7 @@ export const kubernetesConfigBase = () => These are all shared cluster-wide across all users and builds, so they should be resourced accordingly, factoring in how many concurrent builds you expect and how large your images and build contexts tend to be. `), - tlsCertificates: joiArray(tlsCertificateSchema()) + tlsCertificates: joiSparseArray(tlsCertificateSchema()) .unique("name") .description("One or more certificates to use for ingress."), certManager: joi @@ -541,7 +542,7 @@ export const kubernetesConfigBase = () => ) .example({ disktype: "ssd" }) .default(() => ({})), - registryProxyTolerations: joiArray( + registryProxyTolerations: joiSparseArray( joi.object().keys({ effect: joi.string().allow("NoSchedule", "PreferNoSchedule", "NoExecute").description(dedent` "Effect" indicates the taint effect to match. Empty means match all taint effects. When specified, @@ -713,7 +714,7 @@ export const kubernetesTaskSchema = () => .description("The arguments to pass to the container used for execution.") .example(["rake", "db:migrate"]), env: containerEnvVarsSchema(), - artifacts: joiArray(containerArtifactSchema()).description(artifactsDescription), + artifacts: joiSparseArray(containerArtifactSchema()).description(artifactsDescription), }) .description("The task definitions for this module.") @@ -739,7 +740,7 @@ export const kubernetesTestSchema = () => .description("The arguments to pass to the container used for testing.") .example(["npm", "test"]), env: containerEnvVarsSchema(), - artifacts: joiArray(containerArtifactSchema()).description(artifactsDescription), + artifacts: joiSparseArray(containerArtifactSchema()).description(artifactsDescription), }) .description("The test suite definitions for this module.") diff --git a/core/src/plugins/kubernetes/helm/config.ts b/core/src/plugins/kubernetes/helm/config.ts index 417fb60c33..9a0687b2e2 100644 --- a/core/src/plugins/kubernetes/helm/config.ts +++ b/core/src/plugins/kubernetes/helm/config.ts @@ -8,12 +8,12 @@ import { joiPrimitive, - joiArray, joiIdentifier, joiUserIdentifier, DeepPrimitiveMap, joi, joiModuleIncludeDirective, + joiSparseArray, } from "../../../config/common" import { GardenModule } from "../../../types/module" import { containsSource } from "./common" @@ -157,7 +157,7 @@ export const helmModuleSpecSchema = () => Not used when \`base\` is specified.` ) .default("."), - dependencies: joiArray(joiIdentifier()).description( + dependencies: joiSparseArray(joiIdentifier()).description( "List of names of services that should be deployed before this chart." ), namespace: namespaceNameSchema(), @@ -188,8 +188,8 @@ export const helmModuleSpecSchema = () => If neither \`include\` nor \`exclude\` is set and the module specifies a remote chart, Garden automatically sets \`ìnclude\` to \`[]\`. `), - tasks: joiArray(helmTaskSchema()).description("The task definitions for this module."), - tests: joiArray(helmTestSchema()).description("The test suite definitions for this module."), + tasks: joiSparseArray(helmTaskSchema()).description("The task definitions for this module."), + tests: joiSparseArray(helmTestSchema()).description("The test suite definitions for this module."), timeout: joi .number() .integer() @@ -206,7 +206,7 @@ export const helmModuleSpecSchema = () => When specified, these take precedence over the values in the \`values.yaml\` file (or the files specified in \`valueFiles\`). `), - valueFiles: joiArray(joi.posixPath().subPathOnly()).description(dedent` + valueFiles: joiSparseArray(joi.posixPath().subPathOnly()).description(dedent` Specify value files to use when rendering the Helm chart. These will take precedence over the \`values.yaml\` file bundled in the Helm chart, and should be specified in ascending order of precedence. Meaning, the last file in this list will have the highest precedence. diff --git a/core/src/plugins/kubernetes/kubernetes-module/config.ts b/core/src/plugins/kubernetes/kubernetes-module/config.ts index 3730fd77b9..db24d18437 100644 --- a/core/src/plugins/kubernetes/kubernetes-module/config.ts +++ b/core/src/plugins/kubernetes/kubernetes-module/config.ts @@ -7,7 +7,7 @@ */ import { dependenciesSchema } from "../../../config/service" -import { joiArray, joi, joiModuleIncludeDirective } from "../../../config/common" +import { joi, joiModuleIncludeDirective, joiSparseArray } from "../../../config/common" import { GardenModule } from "../../../types/module" import { ConfigureModuleParams, ConfigureModuleResult } from "../../../types/plugin/module/configure" import { Service } from "../../../types/service" @@ -66,18 +66,18 @@ export const kubernetesModuleSpecSchema = () => joi.object().keys({ build: baseBuildSpecSchema(), dependencies: dependenciesSchema(), - manifests: joiArray(kubernetesResourceSchema()).description( + manifests: joiSparseArray(kubernetesResourceSchema()).description( deline` List of Kubernetes resource manifests to deploy. Use this instead of the \`files\` field if you need to resolve template strings in any of the manifests.` ), - files: joiArray(joi.posixPath().subPathOnly()).description( + files: joiSparseArray(joi.posixPath().subPathOnly()).description( "POSIX-style paths to YAML files to load manifests from. Each can contain multiple manifests." ), include: joiModuleIncludeDirective(dedent` - If neither \`include\` nor \`exclude\` is set, Garden automatically sets \`include\` to equal the - \`files\` directive so that only the Kubernetes manifests get included. - `), + If neither \`include\` nor \`exclude\` is set, Garden automatically sets \`include\` to equal the + \`files\` directive so that only the Kubernetes manifests get included. + `), namespace: namespaceNameSchema(), serviceResource: serviceResourceSchema() .description( @@ -90,8 +90,8 @@ export const kubernetesModuleSpecSchema = () => containerModule: containerModuleSchema(), hotReloadArgs: hotReloadArgsSchema(), }), - tasks: joiArray(kubernetesTaskSchema()), - tests: joiArray(kubernetesTestSchema()), + tasks: joiSparseArray(kubernetesTaskSchema()), + tests: joiSparseArray(kubernetesTestSchema()), }) export async function configureKubernetesModule({ diff --git a/core/src/plugins/kubernetes/volumes/persistentvolumeclaim.ts b/core/src/plugins/kubernetes/volumes/persistentvolumeclaim.ts index 8f50144014..e9c45200fb 100644 --- a/core/src/plugins/kubernetes/volumes/persistentvolumeclaim.ts +++ b/core/src/plugins/kubernetes/volumes/persistentvolumeclaim.ts @@ -6,7 +6,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { joiIdentifier, joi, joiArray } from "../../../config/common" +import { joiIdentifier, joi, joiSparseArray } from "../../../config/common" import { dedent } from "../../../util/string" import { BaseVolumeSpec } from "../../base-volume" import { V1PersistentVolumeClaimSpec, V1PersistentVolumeClaim } from "@kubernetes/client-node" @@ -48,7 +48,7 @@ export const pvcModuleDefinition = (): ModuleTypeDefinition => ({ `, schema: joi.object().keys({ build: baseBuildSpecSchema(), - dependencies: joiArray(joiIdentifier()).description( + dependencies: joiSparseArray(joiIdentifier()).description( "List of services and tasks to deploy/run before deploying this PVC." ), namespace: joiIdentifier().description( diff --git a/core/src/plugins/maven-container/maven-container.ts b/core/src/plugins/maven-container/maven-container.ts index fd5856dad0..e24633c784 100644 --- a/core/src/plugins/maven-container/maven-container.ts +++ b/core/src/plugins/maven-container/maven-container.ts @@ -16,7 +16,7 @@ import { ContainerModuleConfig, ContainerTaskSpec, } from "../container/config" -import { joiArray, joiProviderName, joi, joiModuleIncludeDirective } from "../../config/common" +import { joiProviderName, joi, joiModuleIncludeDirective, joiSparseArray } from "../../config/common" import { GardenModule } from "../../types/module" import { resolve } from "path" import { RuntimeError, ConfigurationError } from "../../exceptions" @@ -75,7 +75,7 @@ const mavenKeys = { .description("POSIX-style path to the packaged JAR artifact, relative to the module directory.") .example("target/my-module.jar"), jdkVersion: joi.number().integer().allow(8, 11, 13).default(8).description("The JDK version to use."), - mvnOpts: joiArray(joi.string()).description("Options to add to the `mvn package` command when building."), + mvnOpts: joiSparseArray(joi.string()).description("Options to add to the `mvn package` command when building."), useDefaultDockerfile: joi .boolean() .default(true) diff --git a/core/src/plugins/openfaas/config.ts b/core/src/plugins/openfaas/config.ts index 248b3ac731..5b4aafb393 100644 --- a/core/src/plugins/openfaas/config.ts +++ b/core/src/plugins/openfaas/config.ts @@ -10,7 +10,7 @@ import dedent = require("dedent") import { join } from "path" import { resolve as urlResolve } from "url" import { PluginContext } from "../../plugin-context" -import { joiArray, joiProviderName, joi, joiEnvVars, DeepPrimitiveMap } from "../../config/common" +import { joiProviderName, joi, joiEnvVars, DeepPrimitiveMap, joiSparseArray } from "../../config/common" import { GardenModule } from "../../types/module" import { Service } from "../../types/service" import { ExecModuleSpecBase, ExecTestSpec } from "../exec" @@ -49,7 +49,7 @@ export const openfaasModuleSpecSchema = () => .object() .keys({ build: baseBuildSpecSchema(), - dependencies: joiArray(joi.string()).description( + dependencies: joiSparseArray(joi.string()).description( "The names of services/functions that this function depends on at runtime." ), env: joiEnvVars(), @@ -62,7 +62,7 @@ export const openfaasModuleSpecSchema = () => .string() .description("The image name to use for the built OpenFaaS container (defaults to the module name)"), lang: joi.string().required().description("The OpenFaaS language template to use to build this function."), - tests: joiArray(openfaasTestSchema()).description("A list of tests to run in the module."), + tests: joiSparseArray(openfaasTestSchema()).description("A list of tests to run in the module."), }) .unknown(false) .description("The module specification for an OpenFaaS module.") diff --git a/core/test/unit/src/actions.ts b/core/test/unit/src/actions.ts index fb9dfa7176..89f0801b36 100644 --- a/core/test/unit/src/actions.ts +++ b/core/test/unit/src/actions.ts @@ -151,6 +151,7 @@ describe("ActionRouter", () => { allowPublish: true, build: { dependencies: [] }, disabled: false, + generateFiles: [], }, ], }) diff --git a/core/test/unit/src/config/common.ts b/core/test/unit/src/config/common.ts index 676690356e..70e30e4763 100644 --- a/core/test/unit/src/config/common.ts +++ b/core/test/unit/src/config/common.ts @@ -15,10 +15,25 @@ import { joi, joiRepositoryUrl, joiPrimitive, + joiSparseArray, } from "../../../../src/config/common" import { validateSchema } from "../../../../src/config/validation" import { expectError } from "../../../helpers" +describe("joiSparseArray", () => { + it("should filter out undefined values", () => { + const schema = joiSparseArray(joi.string()).sparse() + const { value } = schema.validate(["foo", undefined, "bar"]) + expect(value).to.eql(["foo", "bar"]) + }) + + it("should filter out null values", () => { + const schema = joiSparseArray(joi.string()).sparse() + const { value } = schema.validate(["foo", undefined, "bar", null, "baz"]) + expect(value).to.eql(["foo", "bar", "baz"]) + }) +}) + describe("envVarRegex", () => { it("should fail on invalid env variables", () => { const testCases = ["GARDEN", "garden", "GARDEN_ENV_VAR", "garden_", "123", ".", "MY-ENV_VAR"] diff --git a/core/test/unit/src/config/service.ts b/core/test/unit/src/config/service.ts new file mode 100644 index 0000000000..9d1ee725d2 --- /dev/null +++ b/core/test/unit/src/config/service.ts @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { expect } from "chai" +import { baseServiceSpecSchema } from "../../../../src/config/service" +import { validateSchema } from "../../../../src/config/validation" + +describe("baseServiceSpecSchema", () => { + it("should filter falsy values from dependencies list", () => { + const input = { + name: "foo", + dependencies: ["service-a", undefined, "service-b", null, "service-c"], + } + const output = validateSchema(input, baseServiceSpecSchema()) + expect(output.dependencies).to.eql(["service-a", "service-b", "service-c"]) + }) +}) diff --git a/core/test/unit/src/config/workflow.ts b/core/test/unit/src/config/workflow.ts index 1c26a9a96a..e0ec3738a7 100644 --- a/core/test/unit/src/config/workflow.ts +++ b/core/test/unit/src/config/workflow.ts @@ -23,6 +23,7 @@ describe("resolveWorkflowConfig", () => { let garden: TestGarden const defaults = { + files: [], limits: defaultContainerLimits, keepAliveHours: 48, } diff --git a/core/test/unit/src/plugins/exec.ts b/core/test/unit/src/plugins/exec.ts index ef4a3e1d10..30fd932e91 100644 --- a/core/test/unit/src/plugins/exec.ts +++ b/core/test/unit/src/plugins/exec.ts @@ -102,6 +102,7 @@ describe("exec plugin", () => { disabled: false, timeout: null, spec: { + artifacts: [], name: "banana", command: ["echo", "BANANA"], env: {}, @@ -117,6 +118,7 @@ describe("exec plugin", () => { disabled: false, timeout: 999, spec: { + artifacts: [], name: "orange", command: ["echo", "ORANGE"], env: {}, @@ -134,6 +136,7 @@ describe("exec plugin", () => { timeout: null, spec: { name: "unit", + artifacts: [], dependencies: [], disabled: false, command: ["echo", "OK"], @@ -162,6 +165,7 @@ describe("exec plugin", () => { timeout: null, spec: { name: "unit", + artifacts: [], dependencies: [], disabled: false, command: ["echo", "OK"], @@ -189,6 +193,7 @@ describe("exec plugin", () => { spec: { name: "unit", dependencies: [], + artifacts: [], disabled: false, command: ["echo", "OK"], env: {}, @@ -217,6 +222,7 @@ describe("exec plugin", () => { name: "pwd", env: {}, command: ["pwd"], + artifacts: [], dependencies: [], disabled: false, timeout: null, diff --git a/docs/reference/module-template-config.md b/docs/reference/module-template-config.md index a41eaae064..ebd72dcaf4 100644 --- a/docs/reference/module-template-config.md +++ b/docs/reference/module-template-config.md @@ -425,9 +425,9 @@ When false, disables pushing this module to remote registries. A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `modules[].generateFiles[].sourcePath` diff --git a/docs/reference/module-types/conftest.md b/docs/reference/module-types/conftest.md index 1fe9661513..be1fdd29dd 100644 --- a/docs/reference/module-types/conftest.md +++ b/docs/reference/module-types/conftest.md @@ -344,9 +344,9 @@ When false, disables pushing this module to remote registries. A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `generateFiles[].sourcePath` diff --git a/docs/reference/module-types/container.md b/docs/reference/module-types/container.md index 1f75b9a682..70dc292b85 100644 --- a/docs/reference/module-types/container.md +++ b/docs/reference/module-types/container.md @@ -730,9 +730,9 @@ When false, disables pushing this module to remote registries. A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `generateFiles[].sourcePath` diff --git a/docs/reference/module-types/exec.md b/docs/reference/module-types/exec.md index 82b5e7d120..473b5826e6 100644 --- a/docs/reference/module-types/exec.md +++ b/docs/reference/module-types/exec.md @@ -455,9 +455,9 @@ When false, disables pushing this module to remote registries. A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `generateFiles[].sourcePath` @@ -580,9 +580,9 @@ Maximum duration (in seconds) of the task's execution. A list of artifacts to copy after the task run. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `tasks[].artifacts[].source` @@ -707,9 +707,9 @@ Key/value map of environment variables. Keys must be valid POSIX environment var A list of artifacts to copy after the test run. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `tests[].artifacts[].source` diff --git a/docs/reference/module-types/hadolint.md b/docs/reference/module-types/hadolint.md index b2d2d50b2e..453a3ee188 100644 --- a/docs/reference/module-types/hadolint.md +++ b/docs/reference/module-types/hadolint.md @@ -332,9 +332,9 @@ When false, disables pushing this module to remote registries. A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `generateFiles[].sourcePath` diff --git a/docs/reference/module-types/helm.md b/docs/reference/module-types/helm.md index a626dde13e..257f15378f 100644 --- a/docs/reference/module-types/helm.md +++ b/docs/reference/module-types/helm.md @@ -636,9 +636,9 @@ When false, disables pushing this module to remote registries. A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `generateFiles[].sourcePath` diff --git a/docs/reference/module-types/kubernetes.md b/docs/reference/module-types/kubernetes.md index f8d568b321..feb1c2d113 100644 --- a/docs/reference/module-types/kubernetes.md +++ b/docs/reference/module-types/kubernetes.md @@ -563,9 +563,9 @@ When false, disables pushing this module to remote registries. A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `generateFiles[].sourcePath` diff --git a/docs/reference/module-types/maven-container.md b/docs/reference/module-types/maven-container.md index 6e700a2e21..2a6113aa21 100644 --- a/docs/reference/module-types/maven-container.md +++ b/docs/reference/module-types/maven-container.md @@ -738,9 +738,9 @@ When false, disables pushing this module to remote registries. A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `generateFiles[].sourcePath` diff --git a/docs/reference/module-types/openfaas.md b/docs/reference/module-types/openfaas.md index a927ddeb98..caad696a44 100644 --- a/docs/reference/module-types/openfaas.md +++ b/docs/reference/module-types/openfaas.md @@ -363,9 +363,9 @@ When false, disables pushing this module to remote registries. A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `generateFiles[].sourcePath` diff --git a/docs/reference/module-types/persistentvolumeclaim.md b/docs/reference/module-types/persistentvolumeclaim.md index c523754d7a..98d3d808d6 100644 --- a/docs/reference/module-types/persistentvolumeclaim.md +++ b/docs/reference/module-types/persistentvolumeclaim.md @@ -385,9 +385,9 @@ When false, disables pushing this module to remote registries. A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `generateFiles[].sourcePath` diff --git a/docs/reference/module-types/templated.md b/docs/reference/module-types/templated.md index 2f9163685b..760d63ecca 100644 --- a/docs/reference/module-types/templated.md +++ b/docs/reference/module-types/templated.md @@ -338,9 +338,9 @@ When false, disables pushing this module to remote registries. A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `generateFiles[].sourcePath` diff --git a/docs/reference/module-types/terraform.md b/docs/reference/module-types/terraform.md index 7759528f6a..5fa61bf5d9 100644 --- a/docs/reference/module-types/terraform.md +++ b/docs/reference/module-types/terraform.md @@ -363,9 +363,9 @@ When false, disables pushing this module to remote registries. A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `generateFiles[].sourcePath` diff --git a/docs/reference/workflow-config.md b/docs/reference/workflow-config.md index 00f50ea367..274e7ecfa0 100644 --- a/docs/reference/workflow-config.md +++ b/docs/reference/workflow-config.md @@ -214,9 +214,9 @@ This is useful to e.g. create files required for provider authentication, and ca Note that you cannot reference provider configuration in template strings within this field, since they are resolved after these files are generated. This means you can reference the files specified here in your provider configurations. -| Type | Required | -| --------------- | -------- | -| `array[object]` | No | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | ### `files[].path` @@ -296,9 +296,9 @@ The maximum amount of RAM the workflow pod can use, in megabytes (i.e. 1024 = 1 The steps the workflow should run. At least one step is required. Steps are run sequentially. If a step fails, subsequent steps are skipped. -| Type | Required | -| --------------- | -------- | -| `array[object]` | Yes | +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | Yes | ### `steps[].name` diff --git a/plugins/conftest/index.ts b/plugins/conftest/index.ts index 8b61d21e62..12aa69c1c2 100644 --- a/plugins/conftest/index.ts +++ b/plugins/conftest/index.ts @@ -17,7 +17,7 @@ import { matchGlobs, listDirectory } from "@garden-io/sdk/util/fs" // TODO: gradually get rid of these core dependencies, move some to SDK etc. import { providerConfigBaseSchema, GenericProviderConfig, Provider } from "@garden-io/core/build/src/config/provider" -import { joi, joiIdentifier, joiArray } from "@garden-io/core/build/src/config/common" +import { joi, joiIdentifier, joiArray, joiSparseArray } from "@garden-io/core/build/src/config/common" import { TestModuleParams } from "@garden-io/core/build/src/types/plugin/module/testModule" import { baseBuildSpecSchema } from "@garden-io/core/build/src/config/module" import { PluginError, ConfigurationError } from "@garden-io/core/build/src/exceptions" @@ -203,7 +203,7 @@ export const gardenPlugin = () => createGardenPlugin({ sourceModule: joiIdentifier() .required() .description("Specify a helm module whose chart we want to test."), - runtimeDependencies: joiArray(joiIdentifier()).description( + runtimeDependencies: joiSparseArray(joiIdentifier()).description( "A list of runtime dependencies that need to be resolved before rendering the Helm chart." ), }), diff --git a/yarn.lock b/yarn.lock index 82980fe722..e1d1828c7b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15399,10 +15399,10 @@ react-transition-group@^4.3.0: loose-envify "^1.4.0" prop-types "^15.6.2" -react-use-gesture@^9.0.0: - version "9.0.4" - resolved "https://registry.yarnpkg.com/react-use-gesture/-/react-use-gesture-9.0.4.tgz#7e0428007df31b6d8daae37c2fb01e9f40283623" - integrity sha512-G0sbQY+HSm2gSVIlD+LE1unpVpG7YZRTr8TI72vo0Nu1lecJtvjbcY3ZLonEZLTtODJgLL6nBllMRXyy0bRSQA== +react-use-gesture@^8.0.1: + version "8.0.1" + resolved "https://registry.yarnpkg.com/react-use-gesture/-/react-use-gesture-8.0.1.tgz#4360c0f7c9e26baf9fbe58f63fc9de7ef699c17f" + integrity sha512-CXzUNkulUdgouaAlvAsC5ZVo0fi9KGSBSk81WrE4kOIcJccpANe9zZkAYr5YZZhqpicIFxitsrGVS4wmoMun9A== react-zlib-js@^1.0.4: version "1.0.5" @@ -15557,10 +15557,10 @@ readdirp@~3.5.0: dependencies: picomatch "^2.2.1" -reaflow@^2.4.4: - version "2.7.0" - resolved "https://registry.yarnpkg.com/reaflow/-/reaflow-2.7.0.tgz#b3709a1139dcf7cb0c85ae00557f7061663a886e" - integrity sha512-CvAEp8FTsh95z3ToqOOHvHnkeccmMDLSQ7/9OExY2uDuBw29wyGJIJ/MHWkgJD1flHdYWtFbkd66VLOKohZ/5w== +reaflow@^3.0.13: + version "3.0.13" + resolved "https://registry.yarnpkg.com/reaflow/-/reaflow-3.0.13.tgz#7e1344cb011aeaf654502f40f80e8f648752b4e2" + integrity sha512-EEPq/4atEpL1RVJxTaLaN8Y3zhYHo2rpLnMbAX7DlI88emJJPqJpTwsH4q5jHspfo2tnatqEe8vmvxXGHLL0mQ== dependencies: calculate-size "^1.1.1" classnames "^2.2.6" @@ -15574,7 +15574,7 @@ reaflow@^2.4.4: rdk "^5.0.6" react-cool-dimensions "^1.2.0" react-fast-compare "^3.2.0" - react-use-gesture "^9.0.0" + react-use-gesture "^8.0.1" reakeys "^1.1.0" undoo "^0.5.0"