feat(k8s): introduce new flag waitForJobs to wait for k8s jobs (#4611)

* feat(kubernetes): wait for kubernetes jobs

* feat(k8s): introduce waitForJobs flag to control jobs behavior

* chore: update docs

* test: fix broken tests

* chore: address pr comments

core/src/plugins/kubernetes/kubernetes-type/config.ts

@@ -46,6 +46,7 @@ export interface KubernetesDeployActionSpec extends KubernetesTypeCommonDeploySp
   defaultTarget?: KubernetesTargetResourceSpec
   sync?: KubernetesDeploySyncSpec
   localMode?: KubernetesLocalModeSpec
+  waitForJobs?: boolean
 }
 
 export type KubernetesDeployActionConfig = DeployActionConfig<"kubernetes", KubernetesDeployActionSpec>
@@ -84,6 +85,13 @@ export const kubernetesCommonDeploySpecKeys = () => ({
   namespace: namespaceNameSchema(),
   portForwards: portForwardsSchema(),
   timeout: k8sDeploymentTimeoutSchema(),
+  // TODO-0.14: flip this to true and change default behavior to
+  // wait for the jobs
+  waitForJobs: joi
+    .boolean()
+    .optional()
+    .default(false)
+    .description("Wait until the jobs have been completed. Garden will wait for as long as `timeout`."),
 })
 
 export const kubernetesDeploySchema = () =>

core/src/plugins/kubernetes/kubernetes-type/handlers.ts

@@ -252,6 +252,7 @@ export const kubernetesDeploy: DeployActionHandler<"deploy", KubernetesDeployAct
       resources: namespaceManifests,
       log,
       timeoutSec: action.getConfig("timeout"),
+      waitForJobs: spec.waitForJobs,
     })
   }
 
@@ -284,6 +285,7 @@ export const kubernetesDeploy: DeployActionHandler<"deploy", KubernetesDeployAct
       resources: preparedManifests,
       log,
       timeoutSec: action.getConfig("timeout"),
+      waitForJobs: spec.waitForJobs,
     })
   }
 

core/src/plugins/kubernetes/status/status.ts

@@ -6,12 +6,12 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
+import Bluebird from "bluebird"
 import { diffString } from "json-diff"
 import { DeploymentError } from "../../../exceptions"
 import { PluginContext } from "../../../plugin-context"
 import { KubeApi } from "../api"
 import { getAppNamespace } from "../namespace"
-import Bluebird from "bluebird"
 import { KubernetesResource, KubernetesServerResource, BaseResource, KubernetesWorkload } from "../types"
 import { zip, isArray, isPlainObject, pickBy, mapValues, flatten, cloneDeep, omit, isEqual, keyBy } from "lodash"
 import { KubernetesProvider, KubernetesPluginContext } from "../config"
@@ -25,6 +25,7 @@ import {
   V1Service,
   V1Container,
   KubernetesObject,
+  V1Job,
 } from "@kubernetes/client-node"
 import dedent = require("dedent")
 import { getPods, getResourceKey, hashManifest } from "../util"
@@ -51,6 +52,7 @@ export interface StatusHandlerParams<T extends BaseResource | KubernetesObject =
   resource: KubernetesServerResource<T>
   log: Log
   resourceVersion?: number
+  waitForJobs?: boolean
 }
 
 interface StatusHandler<T extends BaseResource | KubernetesObject = BaseResource> {
@@ -111,6 +113,37 @@ const objHandlers: { [kind: string]: StatusHandler } = {
 
     return { state: "ready", resource }
   },
+
+  Job: async ({ resource, waitForJobs }: StatusHandlerParams<V1Job>) => {
+    if (
+      resource.status?.failed &&
+      resource.spec?.backoffLimit &&
+      resource.status?.failed >= resource.spec?.backoffLimit
+    ) {
+      // job has failed
+      return { state: "unhealthy", resource }
+    }
+    if (
+      resource.spec?.completions &&
+      resource.status?.succeeded &&
+      resource.status?.succeeded < resource.spec.completions
+    ) {
+      // job is not yet completed
+      return { state: "deploying", resource }
+    }
+    // job has succeeded
+    if (resource.status.succeeded) {
+      return { state: "ready", resource }
+    }
+
+    // wait for job only if waitForJobs is set, otherwise
+    // mark it as ready and proceed.
+    if (waitForJobs) {
+      return { state: "deploying", resource }
+    } else {
+      return { state: "ready", resource }
+    }
+  },
 }
 
 /**
@@ -120,14 +153,21 @@ export async function checkResourceStatuses(
   api: KubeApi,
   namespace: string,
   manifests: KubernetesResource[],
-  log: Log
+  log: Log,
+  waitForJobs?: boolean
 ): Promise<ResourceStatus[]> {
   return Bluebird.map(manifests, async (manifest) => {
-    return checkResourceStatus(api, namespace, manifest, log)
+    return checkResourceStatus(api, namespace, manifest, log, waitForJobs)
   })
 }
 
-export async function checkResourceStatus(api: KubeApi, namespace: string, manifest: KubernetesResource, log: Log) {
+export async function checkResourceStatus(
+  api: KubeApi,
+  namespace: string,
+  manifest: KubernetesResource,
+  log: Log,
+  waitForJobs?: boolean
+) {
   const handler = objHandlers[manifest.kind]
 
   if (manifest.metadata?.namespace) {
@@ -150,7 +190,7 @@ export async function checkResourceStatus(api: KubeApi, namespace: string, manif
 
   let status: ResourceStatus
   if (handler) {
-    status = await handler({ api, namespace, resource, log, resourceVersion })
+    status = await handler({ api, namespace, resource, log, resourceVersion, waitForJobs })
   } else {
     // if there is no explicit handler to check the status, we assume there's no rollout phase to wait for
     status = { state: "ready", resource: manifest }
@@ -167,6 +207,7 @@ interface WaitParams {
   resources: KubernetesResource[]
   log: Log
   timeoutSec: number
+  waitForJobs?: boolean
 }
 
 /**
@@ -180,6 +221,7 @@ export async function waitForResources({
   resources,
   log,
   timeoutSec,
+  waitForJobs,
 }: WaitParams) {
   let loops = 0
   const startTime = new Date().getTime()
@@ -217,7 +259,7 @@ export async function waitForResources({
     await sleep(2000 + 500 * loops)
     loops += 1
 
-    const statuses = await checkResourceStatuses(api, namespace, Object.values(pendingResources), log)
+    const statuses = await checkResourceStatuses(api, namespace, Object.values(pendingResources), log, waitForJobs)
 
     for (const status of statuses) {
       const key = getResourceKey(status.resource)

core/test/integ/src/plugins/kubernetes/kubernetes-module/config.ts

@@ -73,6 +73,7 @@ describe("configureKubernetesModule", () => {
             dependencies: [],
             timeout: DEFAULT_BUILD_TIMEOUT_SEC,
           },
+          waitForJobs: false,
           dependencies: [],
           files: [],
           manifests: [

docs/reference/action-types/Deploy/kubernetes.md

@@ -213,6 +213,9 @@ spec:
   # The maximum duration (in seconds) to wait for resources to deploy and become healthy.
   timeout: 300
 
+  # Wait until the jobs have been completed. Garden will wait for as long as `timeout`.
+  waitForJobs: false
+
   # Specify a default resource in the deployment to use for syncs, local mode, and for the `garden exec` command.
   #
   # Specify either `kind` and `name`, or a `podSelector`. The resource should be one of the resources deployed by this
@@ -818,6 +821,16 @@ The maximum duration (in seconds) to wait for resources to deploy and become hea
 | -------- | ------- | -------- |
 | `number` | `300`   | No       |
 
+### `spec.waitForJobs`
+
+[spec](#spec) > waitForJobs
+
+Wait until the jobs have been completed. Garden will wait for as long as `timeout`.
+
+| Type      | Default | Required |
+| --------- | ------- | -------- |
+| `boolean` | `false` | No       |
+
 ### `spec.defaultTarget`
 
 [spec](#spec) > defaultTarget

docs/reference/module-types/kubernetes.md

@@ -216,6 +216,9 @@ portForwards:
 # The maximum duration (in seconds) to wait for resources to deploy and become healthy.
 timeout: 300
 
+# Wait until the jobs have been completed. Garden will wait for as long as `timeout`.
+waitForJobs: false
+
 # The names of any services that this service depends on at runtime, and the names of any tasks that should be
 # executed before this service is deployed.
 dependencies: []
@@ -995,6 +998,14 @@ The maximum duration (in seconds) to wait for resources to deploy and become hea
 | -------- | ------- | -------- |
 | `number` | `300`   | No       |
 
+### `waitForJobs`
+
+Wait until the jobs have been completed. Garden will wait for as long as `timeout`.
+
+| Type      | Default | Required |
+| --------- | ------- | -------- |
+| `boolean` | `false` | No       |
+
 ### `dependencies[]`
 
 The names of any services that this service depends on at runtime, and the names of any tasks that should be executed before this service is deployed.