Skip to content

Commit

Permalink
fix(k8s): handle intermittent socket hang up errors
Browse files Browse the repository at this point in the history
  • Loading branch information
edvald authored and vvagaytsev committed Jun 7, 2023
1 parent ce545bb commit 51458c6
Showing 1 changed file with 9 additions and 3 deletions.
12 changes: 9 additions & 3 deletions core/src/plugins/kubernetes/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ import { PluginContext } from "../../plugin-context"
import { PassThrough, Readable, Writable } from "stream"
import { getExecExitCode } from "./status/pod"
import { labelSelectorToString } from "./util"
import AsyncLock = require("async-lock")
import AsyncLock from "async-lock"
import request = require("request-promise")
import requestErrors = require("request-promise/errors")

Expand Down Expand Up @@ -998,7 +998,7 @@ async function requestWithRetry<R>(
return await req()
} catch (err) {
if (shouldRetry(err)) {
retryLog = retryLog || log.debug("")
retryLog = retryLog || log.info("")
if (usedRetries <= maxRetries) {
const sleepMsec = minTimeoutMs + usedRetries * minTimeoutMs
retryLog.setState(deline`
Expand Down Expand Up @@ -1039,7 +1039,11 @@ function shouldRetry(err: any): boolean {
code = err.statusCode
}

return (code && statusCodesForRetry.includes(code)) || !!errorMessageRegexesForRetry.find((regex) => msg.match(regex))
return (
(code && statusCodesForRetry.includes(code)) ||
err.code === "ECONNRESET" || // <- socket hang up
!!errorMessageRegexesForRetry.find((regex) => msg.match(regex))
)
}

const statusCodesForRetry: number[] = [
Expand All @@ -1061,6 +1065,8 @@ const errorMessageRegexesForRetry = [
/ETIMEDOUT/,
/ENOTFOUND/,
/EAI_AGAIN/,
/ECONNRESET/,
/socket hang up/,
// This usually isn't retryable
// However on github actions there seems to be flakiness
// And connections get refused temporarily only
Expand Down

0 comments on commit 51458c6

Please sign in to comment.