From 92623b94c32ea8809e2b3e7dfa102b2775c26501 Mon Sep 17 00:00:00 2001 From: stswidwinski Date: Fri, 25 Aug 2023 22:54:58 +0000 Subject: [PATCH] backport of commit f25480c9e929c27476c8930f05832e8b96167660 --- .changelog/18269.txt | 3 +++ client/client.go | 12 ++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 .changelog/18269.txt diff --git a/.changelog/18269.txt b/.changelog/18269.txt new file mode 100644 index 00000000000..435b5b87549 --- /dev/null +++ b/.changelog/18269.txt @@ -0,0 +1,3 @@ +```release-note:bug +client: Ignore stale server updates to prevent GCing allocations that should be running +``` diff --git a/client/client.go b/client/client.go index d5b353fe3ff..21a321dc26d 100644 --- a/client/client.go +++ b/client/client.go @@ -2171,6 +2171,18 @@ OUTER: default: } + // We have not received any new data, or received stale data. This may happen in + // an array of situations, the worst of which seems to be a blocking request + // timeout when the scheduler which we are contacting is newly added or recovering + // after a prolonged downtime. + // + // For full context, please see https://github.com/hashicorp/nomad/issues/18267 + if resp.Index <= req.MinQueryIndex { + c.logger.Debug("Received stale allocation information. Retrying.", + "index", resp.Index, "min_index", req.MinQueryIndex) + continue OUTER + } + // Filter all allocations whose AllocModifyIndex was not incremented. // These are the allocations who have either not been updated, or whose // updates are a result of the client sending an update for the alloc.