improvement(core): increase liveness probes when in hot-reload mode

core/src/plugins/kubernetes/container/deployment.ts

@@ -372,7 +372,7 @@ export async function createWorkloadManifest({
   }
 
   if (spec.healthCheck) {
-    configureHealthCheck(container, spec)
+    configureHealthCheck(container, spec, enableHotReload)
   }
 
   if (spec.volumes && spec.volumes.length) {
@@ -570,7 +570,7 @@ function workloadConfig({
   }
 }
 
-function configureHealthCheck(container: V1Container, spec: ContainerServiceConfig["spec"]): void {
+function configureHealthCheck(container: V1Container, spec: ContainerServiceConfig["spec"], hotReload: boolean): void {
   const readinessPeriodSeconds = 1
   const readinessFailureThreshold = 90
 
@@ -582,16 +582,17 @@ function configureHealthCheck(container: V1Container, spec: ContainerServiceConf
     failureThreshold: readinessFailureThreshold,
   }
 
-  /*
-   * We wait for the effective failure duration (period * threshold) of the readiness probe before starting the
-   * liveness probe.
-   */
+  // We wait for the effective failure duration (period * threshold) of the readiness probe before starting the
+  // liveness probe.
+  // We also increase the periodSeconds and failureThreshold when in hot reload mode. This is to prevent
+  // K8s from restarting the pod when liveness probes fail during build or server restarts on a
+  // hot reload event.
   container.livenessProbe = {
     initialDelaySeconds: readinessPeriodSeconds * readinessFailureThreshold,
-    periodSeconds: 5,
+    periodSeconds: hotReload ? 10 : 5,
     timeoutSeconds: 3,
     successThreshold: 1,
-    failureThreshold: 3,
+    failureThreshold: hotReload ? 30 : 3,
   }
 
   const portsByName = keyBy(spec.ports, "name")

core/test/data/test-projects/container/hot-reload/Dockerfile

@@ -0,0 +1 @@
+FROM busybox:1.31.1

core/test/data/test-projects/container/hot-reload/garden.yml

@@ -0,0 +1,15 @@
+kind: Module
+name: hot-reload
+description: Test module for a simple hot reloadable service
+type: container
+hotReload:
+  sync:
+    - target: /tmp
+services:
+  - name: hot-reload
+    command: [sh, -c, "echo Server running... && nc -l -p 8080"]
+    healthCheck:
+      command: ["echo", "ok"]
+    ports:
+      - name: http
+        containerPort: 8080

core/test/integ/src/plugins/kubernetes/container/deployment.ts

@@ -137,6 +137,37 @@ describe("kubernetes container deployment handlers", () => {
       expect(resource.spec.template?.metadata?.annotations).to.eql(service.spec.annotations)
     })
 
+    it("should increase liveness probes when in hot-reload mode", async () => {
+      const service = graph.getService("hot-reload")
+      const namespace = provider.config.namespace!.name!
+
+      const resource = await createWorkloadManifest({
+        api,
+        provider,
+        service,
+        runtimeContext: emptyRuntimeContext,
+        namespace,
+        enableHotReload: true,
+        log: garden.log,
+        production: false,
+        blueGreen: false,
+      })
+
+      // Find the spec for the actual app container (as opposed to the rsync container)
+      const containerSpec = resource.spec.template?.spec?.containers.find((c) => c.name === "hot-reload")
+
+      expect(containerSpec!.livenessProbe).to.eql({
+        initialDelaySeconds: 90,
+        periodSeconds: 10,
+        timeoutSeconds: 3,
+        successThreshold: 1,
+        failureThreshold: 30,
+        exec: {
+          command: ["echo", "ok"],
+        },
+      })
+    })
+
     it("should name the Deployment with a version suffix and set a version label if blueGreen=true", async () => {
       const service = graph.getService("simple-service")
       const namespace = provider.config.namespace!.name!