From a9f66970eacea093469d71a70ac2372139e18b31 Mon Sep 17 00:00:00 2001 From: Jon Edvald <edvald@gmail.com> Date: Wed, 17 Feb 2021 02:20:38 +0100 Subject: [PATCH] fix(k8s): handle injected service mesh containers for tests+tasks --- core/src/plugins/kubernetes/run.ts | 12 +++-------- core/src/plugins/kubernetes/status/pod.ts | 25 +++++++++++++++-------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/core/src/plugins/kubernetes/run.ts b/core/src/plugins/kubernetes/run.ts index d1fbde17a8..f443e3fb36 100644 --- a/core/src/plugins/kubernetes/run.ts +++ b/core/src/plugins/kubernetes/run.ts @@ -688,13 +688,6 @@ export class PodRunner extends PodRunnerParams { }) } - params.pod.metadata.annotations = { - ...(params.pod.metadata.annotations || {}), - // Workaround to make sure sidecars are not injected, - // due to https://github.com/kubernetes/kubernetes/issues/25908 - "sidecar.istio.io/inject": "false", - } - Object.assign(this, params) this.podName = this.pod.metadata.name @@ -746,7 +739,7 @@ export class PodRunner extends PodRunnerParams { try { await this.createPod({ log, tty }) - // Wait until Pod terminates + // Wait until main container terminates while (true) { const serverPod = await this.api.core.readNamespacedPodStatus(podName, namespace) const state = checkPodStatus(serverPod) @@ -779,7 +772,8 @@ export class PodRunner extends PodRunnerParams { }) } - if (state === "stopped") { + // reason "Completed" means main container is done, but sidecars or other containers possibly still alive + if (state === "stopped" || exitReason === "Completed") { success = exitCode === 0 break } diff --git a/core/src/plugins/kubernetes/status/pod.ts b/core/src/plugins/kubernetes/status/pod.ts index 82c535146a..5e63a98631 100644 --- a/core/src/plugins/kubernetes/status/pod.ts +++ b/core/src/plugins/kubernetes/status/pod.ts @@ -19,12 +19,15 @@ export const POD_LOG_LINES = 30 export function checkPodStatus(pod: KubernetesServerResource<V1Pod>): ServiceState { const phase = pod.status!.phase - if (phase === "Pending") { - // Return "unhealthy" if image or command is invalid + // phase can be "Running" even if some containers have failed, so we need to check container statuses + if (phase === "Pending" || phase === "Running") { const containerStatuses = pod.status!.containerStatuses if (containerStatuses) { + let allTerminated = true + for (const c of containerStatuses) { + // Return "unhealthy" if image or command is invalid if ( c.state && c.state.waiting && @@ -32,21 +35,27 @@ export function checkPodStatus(pod: KubernetesServerResource<V1Pod>): ServiceSta ) { return "unhealthy" } - if (c.state && c.state.terminated) { - if (c.state.terminated.exitCode === 0) { - return "stopped" - } else { + + // One of the containers failed + if (c.state?.terminated) { + if (c.state?.terminated?.exitCode !== 0) { return "unhealthy" } + } else { + allTerminated = false } } + + if (phase === "Running") { + return "ready" + } else if (allTerminated) { + return "stopped" + } } return "deploying" } else if (phase === "Failed") { return "unhealthy" - } else if (phase === "Running") { - return "ready" } else if (phase === "Succeeded" || phase === "Completed") { return "stopped" } else {