feat(k8s): allow setting podSelector on helm/kubernetes resource refs

Previously you could only reference workload resources contained in the
module/chart when choosing which resource to sync to or to use as base
for test/task Pod specs. Now you can reference any Pod by a label
selector, which provides a good deal of flexibility.

core/src/plugins/kubernetes/api.ts

@@ -633,6 +633,14 @@ export class KubeApi {
                   // return the result body directly if applicable
                   .then((res: any) => {
                     if (isPlainObject(res) && res.hasOwnProperty("body")) {
+                      // inexplicably, this API sometimes returns apiVersion and kind as undefined...
+                      if (name === "listNamespacedPod" && res.body.items) {
+                        res.body.items = res.body.items.map((pod: any) => {
+                          pod.apiVersion = "v1"
+                          pod.kind = "Pod"
+                          return pod
+                        })
+                      }
                       return res["body"]
                     } else {
                       return res

core/src/plugins/kubernetes/config.ts

@@ -704,9 +704,10 @@ export const configSchema = () =>
     .unknown(false)
 
 export interface ServiceResourceSpec {
-  kind: HotReloadableKind
+  kind?: HotReloadableKind
   name?: string
   containerName?: string
+  podSelector?: { [key: string]: string }
   containerModule?: string
   hotReloadCommand?: string[]
   hotReloadArgs?: string[]
@@ -729,34 +730,46 @@ export interface KubernetesTestSpec extends BaseTestSpec {
   resource: ServiceResourceSpec
 }
 
+export const serviceResourceDescription = dedent`
+  This can either reference a workload (i.e. a Deployment, DaemonSet or StatefulSet) via the \`kind\` and \`name\` fields, or a Pod via the \`podSelector\` field.
+`
+
 export const serviceResourceSchema = () =>
-  joi.object().keys({
-    // TODO: consider allowing a `resource` field, that includes the kind and name (e.g. Deployment/my-deployment).
-    kind: joi
-      .string()
-      .valid(...hotReloadableKinds)
-      .default("Deployment")
-      .description("The type of Kubernetes resource to sync files to."),
-    name: joi.string().description(
-      deline`The name of the resource to sync to. If the module contains a single resource of the specified Kind,
+  joi
+    .object()
+    .keys({
+      kind: joi
+        .string()
+        .valid(...hotReloadableKinds)
+        .default("Deployment")
+        .description("The type of Kubernetes resource to sync files to."),
+      name: joi.string().description(
+        deline`The name of the resource to sync to. If the module contains a single resource of the specified Kind,
         this can be omitted.`
-    ),
-    containerName: joi.string().description(
-      deline`The name of a container in the target. Specify this if the target contains more than one container
-        and the main container is not the first container in the spec.`
-    ),
-  })
+      ),
+      containerName: joi
+        .string()
+        .description(
+          `The name of a container in the target. Specify this if the target contains more than one container and the main container is not the first container in the spec.`
+        ),
+      podSelector: joiStringMap(joi.string()).description(
+        dedent`
+          A map of string key/value labels to match on any Pods in the namespace. When specified, a random ready Pod with matching labels will be picked as a target, so make sure the labels will always match a specific Pod type.
+        `
+      ),
+    })
+    .oxor("podSelector", "kind")
+    .oxor("podSelector", "name")
 
 export const containerModuleSchema = () =>
   joiIdentifier()
     .description(
-      deline`The Garden module that contains the sources for the container. This needs to be specified under
-    \`serviceResource\` in order to enable hot-reloading, but is not necessary for tasks and tests.
+      dedent`
+        The Garden module that contains the sources for the container. This needs to be specified under \`serviceResource\` in order to enable hot-reloading and dev mode, but is not necessary for tasks and tests.
 
-    Must be a \`container\` module, and for hot-reloading to work you must specify the \`hotReload\` field
-    on the container module.
+        Must be a \`container\` module, and for hot-reloading to work you must specify the \`hotReload\` field on the container module (not required for dev mode).
 
-    Note: If you specify a module here, you don't need to specify it additionally under \`build.dependencies\``
+        _Note: If you specify a module here, you don't need to specify it additionally under \`build.dependencies\`._`
     )
     .example("my-container-module")
 
@@ -773,10 +786,12 @@ export const kubernetesTaskSchema = () =>
   baseTaskSpecSchema()
     .keys({
       resource: serviceResourceSchema().description(
-        dedent`The Deployment, DaemonSet or StatefulSet that Garden should use to execute this task.
+        dedent`The Deployment, DaemonSet, StatefulSet or Pod that Garden should use to execute this task.
         If not specified, the \`serviceResource\` configured on the module will be used. If neither is specified,
         an error will be thrown.
 
+        ${serviceResourceDescription}
+
         The following pod spec fields from the service resource will be used (if present) when executing the task:
         ${runPodSpecWhitelistDescription}`
       ),
@@ -800,10 +815,12 @@ export const kubernetesTestSchema = () =>
   baseTestSpecSchema()
     .keys({
       resource: serviceResourceSchema().description(
-        dedent`The Deployment, DaemonSet or StatefulSet that Garden should use to execute this test suite.
+        dedent`The Deployment, DaemonSet or StatefulSet or Pod that Garden should use to execute this test suite.
         If not specified, the \`serviceResource\` configured on the module will be used. If neither is specified,
         an error will be thrown.
 
+        ${serviceResourceDescription}
+
         The following pod spec fields from the service resource will be used (if present) when executing the test suite:
         ${runPodSpecWhitelistDescription}`
       ),

core/src/plugins/kubernetes/dev-mode.ts

@@ -10,7 +10,7 @@ const AsyncLock = require("async-lock")
 import { containerDevModeSchema, ContainerDevModeSpec } from "../container/config"
 import { dedent, gardenAnnotationKey } from "../../util/string"
 import { fromPairs, set } from "lodash"
-import { getResourceContainer } from "./util"
+import { getResourceContainer, getResourcePodSpec } from "./util"
 import { HotReloadableResource } from "./hot-reload/hot-reload"
 import { LogEntry } from "../../logger/log-entry"
 import { joinWithPosix } from "../../util/fs"
@@ -80,18 +80,24 @@ export function configureDevMode({ target, spec, containerName }: ConfigureDevMo
     return
   }
 
+  const podSpec = getResourcePodSpec(target)
+
+  if (!podSpec) {
+    return
+  }
+
   // Inject mutagen agent on init
   const gardenVolumeName = `garden`
   const gardenVolumeMount = {
     name: gardenVolumeName,
     mountPath: "/.garden",
   }
 
-  if (!target.spec.template.spec!.volumes) {
-    target.spec.template.spec!.volumes = []
+  if (!podSpec.volumes) {
+    podSpec.volumes = []
   }
 
-  target.spec.template.spec!.volumes.push({
+  podSpec.volumes.push({
     name: gardenVolumeName,
     emptyDir: {},
   })
@@ -105,10 +111,10 @@ export function configureDevMode({ target, spec, containerName }: ConfigureDevMo
     volumeMounts: [gardenVolumeMount],
   }
 
-  if (!target.spec.template.spec!.initContainers) {
-    target.spec.template.spec!.initContainers = []
+  if (!podSpec.initContainers) {
+    podSpec.initContainers = []
   }
-  target.spec.template.spec!.initContainers.push(initContainer)
+  podSpec.initContainers.push(initContainer)
 
   if (!mainContainer.volumeMounts) {
     mainContainer.volumeMounts = []
@@ -160,7 +166,7 @@ export async function startDevModeSync({
     }
 
     if (!containerName) {
-      containerName = target.spec.template.spec?.containers[0]?.name
+      containerName = getResourcePodSpec(target)?.containers[0]?.name
     }
 
     if (!containerName) {

core/src/plugins/kubernetes/helm/config.ts

@@ -33,6 +33,7 @@ import {
   namespaceNameSchema,
   containerModuleSchema,
   hotReloadArgsSchema,
+  serviceResourceDescription,
 } from "../config"
 import { posix } from "path"
 import { runPodSpecIncludeFields } from "../run"
@@ -104,10 +105,12 @@ const runPodSpecWhitelistDescription = runPodSpecIncludeFields.map((f) => `* \`$
 const helmTaskSchema = () =>
   kubernetesTaskSchema().keys({
     resource: helmServiceResourceSchema().description(
-      dedent`The Deployment, DaemonSet or StatefulSet that Garden should use to execute this task.
+      dedent`The Deployment, DaemonSet or StatefulSet or Pod that Garden should use to execute this task.
         If not specified, the \`serviceResource\` configured on the module will be used. If neither is specified,
         an error will be thrown.
 
+        ${serviceResourceDescription}
+
         The following pod spec fields from the service resource will be used (if present) when executing the task:
         ${runPodSpecWhitelistDescription}`
     ),
@@ -116,10 +119,12 @@ const helmTaskSchema = () =>
 const helmTestSchema = () =>
   kubernetesTestSchema().keys({
     resource: helmServiceResourceSchema().description(
-      dedent`The Deployment, DaemonSet or StatefulSet that Garden should use to execute this test suite.
+      dedent`The Deployment, DaemonSet or StatefulSet or Pod that Garden should use to execute this test suite.
         If not specified, the \`serviceResource\` configured on the module will be used. If neither is specified,
         an error will be thrown.
 
+        ${serviceResourceDescription}
+
         The following pod spec fields from the service resource will be used (if present) when executing the test suite:
         ${runPodSpecWhitelistDescription}`
     ),
@@ -170,13 +175,13 @@ export const helmModuleSpecSchema = () =>
     ),
     repo: joi.string().description("The repository URL to fetch the chart from."),
     serviceResource: helmServiceResourceSchema().description(
-      deline`The Deployment, DaemonSet or StatefulSet that Garden should regard as the _Garden service_ in this module
-      (not to be confused with Kubernetes Service resources).
-      Because a Helm chart can contain any number of Kubernetes resources, this needs to be specified for certain
-      Garden features and commands to work, such as hot-reloading.
+      dedent`
+      The Deployment, DaemonSet or StatefulSet or Pod that Garden should regard as the _Garden service_ in this module (not to be confused with Kubernetes Service resources).
+
+      ${serviceResourceDescription}
 
-      We currently map a Helm chart to a single Garden service, because all the resources in a Helm chart are
-      deployed at once.`
+      Because a Helm chart can contain any number of Kubernetes resources, this needs to be specified for certain Garden features and commands to work, such as hot-reloading.
+      `
     ),
     skipDeploy: joi
       .boolean()

core/src/plugins/kubernetes/helm/deployment.ts

@@ -18,7 +18,7 @@ import { ContainerHotReloadSpec } from "../../container/config"
 import { DeployServiceParams } from "../../../types/plugin/service/deployService"
 import { DeleteServiceParams } from "../../../types/plugin/service/deleteService"
 import { getForwardablePorts, killPortForwards } from "../port-forward"
-import { findServiceResource, getServiceResourceSpec } from "../util"
+import { getServiceResource, getServiceResourceSpec } from "../util"
 import { getModuleNamespace, getModuleNamespaceStatus } from "../namespace"
 import { getHotReloadSpec, configureHotReload, getHotReloadContainerName } from "../hot-reload/helpers"
 import { configureDevMode, startDevModeSync } from "../dev-mode"
@@ -40,13 +40,22 @@ export async function deployHelmService({
   const k8sCtx = ctx as KubernetesPluginContext
   const provider = k8sCtx.provider
 
+  const namespaceStatus = await getModuleNamespaceStatus({
+    ctx: k8sCtx,
+    log,
+    module,
+    provider: k8sCtx.provider,
+  })
+  const namespace = namespaceStatus.namespaceName
+
   const manifests = await getChartResources({ ctx: k8sCtx, module, devMode, hotReload, log, version: service.version })
 
   if ((devMode && module.spec.devMode) || hotReload) {
     serviceResourceSpec = getServiceResourceSpec(module, getBaseModule(module))
-    serviceResource = await findServiceResource({
+    serviceResource = await getServiceResource({
       ctx,
       log,
+      provider,
       module,
       manifests,
       resourceSpec: serviceResourceSpec,
@@ -59,14 +68,6 @@ export async function deployHelmService({
 
   const chartPath = await getChartPath(module)
 
-  const namespaceStatus = await getModuleNamespaceStatus({
-    ctx: k8sCtx,
-    log,
-    module,
-    provider: k8sCtx.provider,
-  })
-  const namespace = namespaceStatus.namespaceName
-
   const releaseName = getReleaseName(module)
   const releaseStatus = await getReleaseStatus({ ctx: k8sCtx, service, releaseName, log, devMode, hotReload })
 

core/src/plugins/kubernetes/helm/exec.ts

@@ -10,7 +10,7 @@ import { includes } from "lodash"
 import { DeploymentError } from "../../../exceptions"
 import { getAppNamespace } from "../namespace"
 import { KubernetesPluginContext } from "../config"
-import { execInWorkload, findServiceResource, getServiceResourceSpec } from "../util"
+import { execInWorkload, getServiceResource, getServiceResourceSpec } from "../util"
 import { ExecInServiceParams } from "../../../types/plugin/service/execInService"
 import { HelmModule } from "./config"
 import { getServiceStatus } from "./status"
@@ -44,9 +44,10 @@ export async function execInHelmService(params: ExecInServiceParams<HelmModule>)
     version: service.version,
   })
 
-  const serviceResource = await findServiceResource({
+  const serviceResource = await getServiceResource({
     ctx,
     log,
+    provider,
     module,
     manifests,
     resourceSpec: serviceResourceSpec,

core/src/plugins/kubernetes/helm/run.ts

@@ -10,7 +10,7 @@ import { HelmModule } from "./config"
 import { PodRunner, runAndCopy } from "../run"
 import { getChartResources, getBaseModule } from "./common"
 import {
-  findServiceResource,
+  getServiceResource,
   getResourceContainer,
   getResourcePodSpec,
   getServiceResourceSpec,
@@ -61,9 +61,10 @@ export async function runHelmModule({
   }
 
   const manifests = await getChartResources({ ctx: k8sCtx, module, devMode: false, hotReload: false, log, version })
-  const target = await findServiceResource({
+  const target = await getServiceResource({
     ctx: k8sCtx,
     log,
+    provider: k8sCtx.provider,
     manifests,
     module,
     resourceSpec,
@@ -133,9 +134,10 @@ export async function runHelmTask(params: RunTaskParams<HelmModule>): Promise<Ru
   })
   const baseModule = getBaseModule(module)
   const resourceSpec = task.spec.resource || getServiceResourceSpec(module, baseModule)
-  const target = await findServiceResource({
+  const target = await getServiceResource({
     ctx: k8sCtx,
     log,
+    provider: k8sCtx.provider,
     manifests,
     module,
     resourceSpec,

core/src/plugins/kubernetes/helm/status.ts

@@ -16,7 +16,7 @@ import { KubernetesPluginContext } from "../config"
 import { getForwardablePorts } from "../port-forward"
 import { KubernetesServerResource } from "../types"
 import { getModuleNamespace, getModuleNamespaceStatus } from "../namespace"
-import { findServiceResource, getServiceResourceSpec } from "../util"
+import { getServiceResource, getServiceResourceSpec } from "../util"
 import chalk from "chalk"
 import { startDevModeSync } from "../dev-mode"
 import { gardenAnnotationKey } from "../../../util/string"
@@ -74,9 +74,10 @@ export async function getServiceStatus({
       // Need to start the dev-mode sync here, since the deployment handler won't be called.
       const baseModule = getBaseModule(module)
       const serviceResourceSpec = getServiceResourceSpec(module, baseModule)
-      const target = await findServiceResource({
-        ctx,
+      const target = await getServiceResource({
+        ctx: k8sCtx,
         log,
+        provider: k8sCtx.provider,
         module,
         manifests: deployedResources,
         resourceSpec: serviceResourceSpec,

core/src/plugins/kubernetes/helm/test.ts

@@ -16,7 +16,7 @@ import { TestModuleParams } from "../../../types/plugin/module/testModule"
 import { TestResult } from "../../../types/plugin/module/getTestResult"
 import {
   getServiceResourceSpec,
-  findServiceResource,
+  getServiceResource,
   getResourceContainer,
   makePodName,
   getResourcePodSpec,
@@ -38,7 +38,14 @@ export async function testHelmModule(params: TestModuleParams<HelmModule>): Prom
   })
   const baseModule = getBaseModule(module)
   const resourceSpec = test.config.spec.resource || getServiceResourceSpec(module, baseModule)
-  const target = await findServiceResource({ ctx: k8sCtx, log, manifests, module, resourceSpec })
+  const target = await getServiceResource({
+    ctx: k8sCtx,
+    log,
+    provider: k8sCtx.provider,
+    manifests,
+    module,
+    resourceSpec,
+  })
   const container = getResourceContainer(target, resourceSpec.containerName)
   const namespaceStatus = await getModuleNamespaceStatus({
     ctx: k8sCtx,