Skip to content

Commit

Permalink
fix(k8s): don't error on pod scheduling warning
Browse files Browse the repository at this point in the history
This fixes an issue where, for example, a Pod could fail to schedule
while a PersistentVolumeClaim is resolving.
  • Loading branch information
edvald committed Feb 22, 2019
1 parent 7e26135 commit 49f8115
Showing 1 changed file with 20 additions and 6 deletions.
26 changes: 20 additions & 6 deletions garden-service/src/plugins/kubernetes/status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { DeploymentError } from "../../exceptions"
import { PluginContext } from "../../plugin-context"
import { ServiceState, combineStates } from "../../types/service"
import { sleep } from "../../util/util"
import { KubeApi } from "./api"
import { KubeApi, KubernetesError } from "./api"
import { KUBECTL_DEFAULT_TIMEOUT } from "./kubectl"
import { getAppNamespace } from "./namespace"
import * as Bluebird from "bluebird"
Expand Down Expand Up @@ -173,6 +173,12 @@ export async function checkDeploymentStatus(
}

if (event.type === "Warning" || event.type === "Error") {
if (event.type === "Warning" && event.reason === "FailedScheduling") {
// this can happen on first attempt to schedule a pod
// TODO: we may want to more specifically look at the message
// (e.g. 'pod has unbound immediate PersistentVolumeClaims')
continue
}
if (event.reason === "Unhealthy") {
// still waiting on readiness probe
continue
Expand Down Expand Up @@ -561,10 +567,18 @@ async function getPodLogs(api: KubeApi, namespace: string, podNames: string[]):
const allLogs = await Bluebird.map(podNames, async (name) => {
// Putting 5000 bytes as a length limit in addition to the line limit, just as a precaution in case someone
// accidentally logs a binary file or something.
const res = await api.core.readNamespacedPodLog(
name, namespace, undefined, false, 5000, undefined, false, undefined, podLogLines,
)
return `****** ${name} ******\n${res.body}`
try {
const res = await api.core.readNamespacedPodLog(
name, namespace, undefined, false, 5000, undefined, false, undefined, podLogLines,
)
return res.body ? `****** ${name} ******\n${res.body}` : ""
} catch (err) {
if (err instanceof KubernetesError && err.message.includes("waiting to start")) {
return ""
} else {
throw err
}
}
})
return allLogs.join("\n\n")
return allLogs.filter(l => l !== "").join("\n\n")
}

0 comments on commit 49f8115

Please sign in to comment.