feat(config): allow sparse arrays where appropriate in config schemas

Users can now place undefined or null values in place of real values
in many array configuration values, in order to conditionally place
values there. Generally, this applies for any array schema that wouldn't
naturally expect null values (which would make the intent ambiguous).

For example, you might do something like
`dependencies: ["service-a", ${var.some-condition ? 'service-b' : null}]`
to conditionally include a service dependency.

core/src/config/common.ts

@@ -119,6 +119,7 @@ export interface Schema extends Joi.Root {
   gitUrl: () => GitUrlSchema
   posixPath: () => PosixPathSchema
   hostname: () => Joi.StringSchema
+  sparseArray: () => Joi.ArraySchema
 }
 
 export let joi: Schema = Joi.extend({
@@ -393,6 +394,19 @@ joi = joi.extend({
   },
 })
 
+/**
+ * Add a joi.sparseArray() type, that both allows sparse arrays _and_ filters the falsy values out.
+ */
+joi = joi.extend({
+  base: Joi.array().sparse(true),
+  type: "sparseArray",
+  coerce: {
+    method(value) {
+      return { value: value && value.filter((v: any) => v !== undefined && v !== null) }
+    },
+  },
+})
+
 export const joiPrimitive = () =>
   joi
     .alternatives()
@@ -482,6 +496,9 @@ export const joiEnvVars = () =>
 
 export const joiArray = (schema: Joi.Schema) => joi.array().items(schema).default([])
 
+// This allows null, empty string or undefined values on the item values and then filters them out
+export const joiSparseArray = (schema: Joi.Schema) => joi.sparseArray().items(schema.allow(null)).default([])
+
 export const joiRepositoryUrl = () =>
   joi
     .alternatives(

core/src/config/module.ts

@@ -17,6 +17,7 @@ import {
   apiVersionSchema,
   DeepPrimitiveMap,
   joiVariables,
+  joiSparseArray,
 } from "./common"
 import { TestConfig, testConfigSchema } from "./test"
 import { TaskConfig, taskConfigSchema } from "./task"
@@ -55,7 +56,7 @@ export const buildDependencySchema = () =>
   joi.object().keys({
     name: joi.string().required().description("Module name to build ahead of this module."),
     plugin: joi.string().meta({ internal: true }).description("The name of plugin that provides the build dependency."),
-    copy: joiArray(copySchema()).description(
+    copy: joiSparseArray(copySchema()).description(
       "Specify one or more files or directories to copy from the built dependency to this module."
     ),
   })
@@ -132,7 +133,7 @@ export const baseBuildSpecSchema = () =>
   joi
     .object()
     .keys({
-      dependencies: joiArray(buildDependencySchema())
+      dependencies: joiSparseArray(buildDependencySchema())
         .description("A list of modules that must be built before this module is built.")
         .example([{ name: "some-other-module-name" }]),
     })
@@ -206,7 +207,7 @@ export const baseModuleSpecKeys = () => ({
     .boolean()
     .default(true)
     .description("When false, disables pushing this module to remote registries."),
-  generateFiles: joi.array().items(generatedFileSchema()).description(dedent`
+  generateFiles: joiSparseArray(generatedFileSchema()).description(dedent`
     A list of files to write to the module directory when resolving this module. This is useful to automatically generate (and template) any supporting files needed for the module.
   `),
 })

core/src/config/project.ts

@@ -22,6 +22,7 @@ import {
   DeepPrimitiveMap,
   joiVariablesDescription,
   apiVersionSchema,
+  joiSparseArray,
 } from "./common"
 import { validateWithPath } from "./validation"
 import { resolveTemplateStrings } from "../template-string"
@@ -134,9 +135,9 @@ export const environmentSchema = () =>
 export const environmentsSchema = () =>
   joi
     .alternatives(
-      joi.array().items(environmentSchema()).unique("name"),
+      joiSparseArray(environmentSchema()).unique("name"),
       // Allow a string as a shorthand for { name: foo }
-      joi.array().items(joiUserIdentifier())
+      joiSparseArray(joiUserIdentifier())
     )
     .description("A list of environments to configure for the project.")
 
@@ -158,7 +159,7 @@ export const projectSourceSchema = () =>
   })
 
 export const projectSourcesSchema = () =>
-  joiArray(projectSourceSchema()).unique("name").description("A list of remote sources to import into project.")
+  joiSparseArray(projectSourceSchema()).unique("name").description("A list of remote sources to import into project.")
 
 export const linkedSourceSchema = () =>
   joi.object().keys({
@@ -296,7 +297,7 @@ export const projectDocsSchema = () =>
         .array()
         .items(environmentSchema())
         .description((<any>environmentsSchema().describe().flags).description),
-      providers: joiArray(providerConfigBaseSchema()).description(
+      providers: joiSparseArray(providerConfigBaseSchema()).description(
         "A list of providers that should be used for this project, and their configuration. " +
           "Please refer to individual plugins/providers for details on how to configure them."
       ),
@@ -313,7 +314,7 @@ export const projectDocsSchema = () =>
           `
         )
         .example("dev"),
-      dotIgnoreFiles: joiArray(joi.posixPath().filenameOnly())
+      dotIgnoreFiles: joiSparseArray(joi.posixPath().filenameOnly())
         .default(defaultDotIgnoreFiles)
         .description(
           deline`
@@ -326,7 +327,7 @@ export const projectDocsSchema = () =>
         )
         .example([".gardenignore", ".gitignore"]),
       modules: projectModulesSchema().description("Control where to scan for modules in the project."),
-      outputs: joiArray(projectOutputSchema())
+      outputs: joiSparseArray(projectOutputSchema())
         .unique("name")
         .description(
           dedent`

core/src/config/provider.ts

@@ -7,7 +7,7 @@
  */
 
 import { deline } from "../util/string"
-import { joiIdentifier, joiUserIdentifier, joiArray, joi, joiIdentifierMap } from "./common"
+import { joiIdentifier, joiUserIdentifier, joiArray, joi, joiIdentifierMap, joiSparseArray } from "./common"
 import { collectTemplateReferences } from "../template-string"
 import { ConfigurationError } from "../exceptions"
 import { ModuleConfig, moduleConfigSchema } from "./module"
@@ -30,7 +30,7 @@ export interface GenericProviderConfig extends BaseProviderConfig {
 const providerFixedFieldsSchema = () =>
   joi.object().keys({
     name: joiIdentifier().required().description("The name of the provider plugin to use.").example("local-kubernetes"),
-    dependencies: joiArray(joiIdentifier())
+    dependencies: joiSparseArray(joiIdentifier())
       .description("List other providers that should be resolved before this one.")
       .example(["exec"]),
     environments: joi

core/src/config/service.ts

@@ -6,7 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-import { joiIdentifier, joiArray, joiUserIdentifier, joi, joiVariables } from "./common"
+import { joiIdentifier, joiUserIdentifier, joi, joiVariables, joiSparseArray } from "./common"
 import { deline, dedent } from "../util/string"
 
 /**
@@ -22,7 +22,7 @@ export interface CommonServiceSpec {
 export const serviceOutputsSchema = joiVariables()
 
 export const dependenciesSchema = () =>
-  joiArray(joiIdentifier()).description(deline`
+  joiSparseArray(joiIdentifier()).description(deline`
     The names of any services that this service depends on at runtime, and the names of any
     tasks that should be executed before this service is deployed.
   `)

core/src/config/task.ts

@@ -6,7 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-import { joiArray, joiUserIdentifier, joi } from "./common"
+import { joiUserIdentifier, joi, joiSparseArray } from "./common"
 import { deline, dedent } from "../util/string"
 
 export interface TaskSpec {}
@@ -35,7 +35,7 @@ export const baseTaskSpecSchema = () =>
     .keys({
       name: joiUserIdentifier().required().description("The name of the task."),
       description: joi.string().optional().description("A description of the task."),
-      dependencies: joiArray(joi.string()).description(deline`
+      dependencies: joiSparseArray(joi.string()).description(deline`
         The names of any tasks that must be executed, and the names of any services that must be running, before this task is executed.
       `),
       disabled: joi

core/src/config/test.ts

@@ -6,7 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-import { joiArray, joiUserIdentifier, joi } from "./common"
+import { joiUserIdentifier, joi, joiSparseArray } from "./common"
 import { deline, dedent } from "../util/string"
 
 export interface BaseTestSpec {
@@ -19,7 +19,7 @@ export interface BaseTestSpec {
 export const baseTestSpecSchema = () =>
   joi.object().keys({
     name: joiUserIdentifier().required().description("The name of the test."),
-    dependencies: joiArray(joi.string()).description(deline`
+    dependencies: joiSparseArray(joi.string()).description(deline`
         The names of any services that must be running, and the names of any
         tasks that must be executed, before the test is run.
       `),

core/src/config/workflow.ts

@@ -7,7 +7,15 @@
  */
 
 import { isEqual, merge, omit, take } from "lodash"
-import { joi, joiUserIdentifier, joiVariableName, joiIdentifier, joiEnvVars, PrimitiveMap } from "./common"
+import {
+  joi,
+  joiUserIdentifier,
+  joiVariableName,
+  joiIdentifier,
+  joiEnvVars,
+  PrimitiveMap,
+  joiSparseArray,
+} from "./common"
 import { DEFAULT_API_VERSION } from "../constants"
 import { deline, dedent } from "../util/string"
 import { defaultContainerLimits, ServiceLimitSpec } from "../plugins/container/config"
@@ -64,7 +72,7 @@ export const workflowConfigSchema = () =>
       kind: joi.string().default("Workflow").valid("Workflow"),
       name: joiUserIdentifier().required().description("The name of this workflow.").example("my-workflow"),
       description: joi.string().description("A description of the workflow."),
-      files: joi.array().items(workflowFileSchema()).description(dedent`
+      files: joiSparseArray(workflowFileSchema()).description(dedent`
           A list of files to write before starting the workflow.
 
           This is useful to e.g. create files required for provider authentication, and can be created from data stored in secrets or templated strings.
@@ -90,7 +98,7 @@ export const workflowConfigSchema = () =>
             .description("The maximum amount of RAM the workflow pod can use, in megabytes (i.e. 1024 = 1 GB)"),
         })
         .default(defaultContainerLimits),
-      steps: joi.array().items(workflowStepSchema()).required().min(1).description(deline`
+      steps: joiSparseArray(workflowStepSchema()).required().min(1).description(deline`
           The steps the workflow should run. At least one step is required. Steps are run sequentially.
           If a step fails, subsequent steps are skipped.
         `),

core/src/docs/common.ts

@@ -124,3 +124,7 @@ export function getGitHubUrl(path: string) {
 export function templateStringLiteral(key: string) {
   return "`${" + key + "}`"
 }
+
+export function isArrayType(type: string) {
+  return type === "array" || type === "sparseArray"
+}

core/src/docs/config.ts

@@ -22,6 +22,7 @@ import {
   BaseKeyDescription,
   NormalizeOptions,
   flattenSchema,
+  isArrayType,
 } from "./common"
 import { JoiDescription, JoiKeyDescription } from "./joi-schema"
 import { safeDumpYaml } from "../util/util"
@@ -143,9 +144,9 @@ export function renderSchemaDescriptionYaml(
     const comment: string[] = []
     const out: string[] = []
     const isFirstChild = parent && parent === prevDesc
-    const isArrayItem = parent && parent.type === "array"
+    const isArrayItem = parent && isArrayType(parent.type)
     const isFirstArrayItem = isArrayItem && isFirstChild
-    const isPrimitive = type !== "array" && type !== "object"
+    const isPrimitive = !isArrayType(type) && type !== "object"
 
     const presetValue = getPresetValue(desc)
 

core/src/docs/joi-schema.ts

@@ -8,7 +8,7 @@
 
 import Joi from "@hapi/joi"
 import { uniq, isFunction, extend, isArray, isPlainObject } from "lodash"
-import { BaseKeyDescription } from "./common"
+import { BaseKeyDescription, isArrayType } from "./common"
 import { findByName, safeDumpYaml } from "../util/util"
 import { JsonKeyDescription } from "./json-schema"
 
@@ -63,7 +63,7 @@ export class JoiKeyDescription extends BaseKeyDescription {
   }
 
   formatName() {
-    return this.type === "array" ? `${this.name}[]` : this.name
+    return isArrayType(this.type) ? `${this.name}[]` : this.name
   }
 
   formatExample() {
@@ -139,7 +139,7 @@ export class JoiKeyDescription extends BaseKeyDescription {
         )
       }
       return childDescriptions
-    } else if (this.joiDescription.type === "array" && this.joiDescription.items[0]) {
+    } else if (isArrayType(this.joiDescription.type) && this.joiDescription.items[0]) {
       // We only use the first array item
       return [
         new JoiKeyDescription({
@@ -176,7 +176,7 @@ function getObjectSchema(d: JoiDescription) {
 
 function formatType(joiDescription: JoiDescription) {
   const { type } = joiDescription
-  const items = type === "array" && joiDescription.items
+  const items = isArrayType(type) && joiDescription.items
 
   if (items && items.length > 0) {
     // We don't consider an array of primitives as children

core/src/plugins/container/config.ts

@@ -9,14 +9,14 @@
 import { GardenModule, FileCopySpec } from "../../types/module"
 import {
   joiUserIdentifier,
-  joiArray,
   PrimitiveMap,
   joiPrimitive,
   joi,
   envVarRegex,
   Primitive,
   joiModuleIncludeDirective,
   joiIdentifier,
+  joiSparseArray,
 } from "../../config/common"
 import { ArtifactSpec } from "./../../config/validation"
 import { Service, ingressHostnameSchema, linkUrlSchema } from "../../types/service"
@@ -337,7 +337,7 @@ const volumeSchema = () =>
     .oxor("hostPath", "module")
 
 export function getContainerVolumesSchema(targetType: string) {
-  return joiArray(volumeSchema()).unique("name").description(dedent`
+  return joiSparseArray(volumeSchema()).unique("name").description(dedent`
     List of volumes that should be mounted when deploying the ${targetType}.
 
     Note: If neither \`hostPath\` nor \`module\` is specified, an empty ephemeral volume is created and mounted when deploying the container.
@@ -363,7 +363,7 @@ const containerServiceSchema = () =>
         Whether to run the service as a daemon (to ensure exactly one instance runs per node).
         May not be supported by all providers.
       `),
-    ingresses: joiArray(ingressSchema())
+    ingresses: joiSparseArray(ingressSchema())
       .description("List of ingress endpoints that the service exposes.")
       .example([{ path: "/api", port: "http" }]),
     env: containerEnvVarsSchema(),
@@ -387,7 +387,7 @@ const containerServiceSchema = () =>
       )
       .example(["npm", "run", "dev"]),
     limits: limitsSchema().description("Specify resource limits for the service.").default(defaultContainerLimits),
-    ports: joiArray(portSchema()).unique("name").description("List of ports that the service container exposes."),
+    ports: joiSparseArray(portSchema()).unique("name").description("List of ports that the service container exposes."),
     replicas: joi.number().integer().description(deline`
       The number of instances of the service to deploy.
       Defaults to 3 for environments configured with \`production: true\`, otherwise 1.
@@ -584,12 +584,12 @@ export const containerModuleSpecSchema = () =>
     `),
       hotReload: hotReloadConfigSchema(),
       dockerfile: joi.posixPath().subPathOnly().description("POSIX-style name of Dockerfile, relative to module root."),
-      services: joiArray(containerServiceSchema())
+      services: joiSparseArray(containerServiceSchema())
         .unique("name")
         .description("A list of services to deploy from this container module."),
-      tests: joiArray(containerTestSchema()).description("A list of tests to run in the module."),
+      tests: joiSparseArray(containerTestSchema()).description("A list of tests to run in the module."),
       // We use the user-facing term "tasks" as the key here, instead of "tasks".
-      tasks: joiArray(containerTaskSchema()).description(deline`
+      tasks: joiSparseArray(containerTaskSchema()).description(deline`
         A list of tasks that can be run from this container module. These can be used as dependencies for services
         (executed before the service is deployed) or for other tasks.
       `),

core/src/plugins/exec.ts

@@ -10,7 +10,7 @@ import Bluebird from "bluebird"
 import { mapValues } from "lodash"
 import { join } from "path"
 import cpy = require("cpy")
-import { joiArray, joiEnvVars, joi } from "../config/common"
+import { joiArray, joiEnvVars, joi, joiSparseArray } from "../config/common"
 import { validateWithPath, ArtifactSpec } from "../config/validation"
 import { createGardenPlugin } from "../types/plugin/plugin"
 import { GardenModule, getModuleKey } from "../types/module"
@@ -52,7 +52,7 @@ const artifactSchema = () =>
     target: joi.posixPath().relativeOnly().subPathOnly().default(".").description(artifactsTargetDescription),
   })
 
-const artifactsSchema = () => joi.array().items(artifactSchema())
+const artifactsSchema = () => joiSparseArray(artifactSchema())
 
 export interface ExecTestSpec extends BaseTestSpec {
   command: string[]
@@ -152,8 +152,8 @@ export const execModuleSpecSchema = () =>
         .default(false),
       build: execBuildSpecSchema(),
       env: joiEnvVars(),
-      tasks: joiArray(execTaskSpecSchema()).description("A list of tasks that can be run in this module."),
-      tests: joiArray(execTestSchema()).description("A list of tests to run in the module."),
+      tasks: joiSparseArray(execTaskSpecSchema()).description("A list of tasks that can be run in this module."),
+      tests: joiSparseArray(execTestSchema()).description("A list of tests to run in the module."),
     })
     .unknown(false)
     .description("The module specification for an exec module.")