From 82e5d3cf51e70953731c4e6c4ed42d05fa5daecc Mon Sep 17 00:00:00 2001
From: hc-github-team-nomad-core
 <82989552+hc-github-team-nomad-core@users.noreply.github.com>
Date: Fri, 25 Aug 2023 18:35:47 -0500
Subject: [PATCH] backport of commit f25480c9e929c27476c8930f05832e8b96167660
 (#18339)

Co-authored-by: stswidwinski <stan.swidwinski@gmail.com>
---
 .changelog/18269.txt |  3 +++
 client/client.go     | 12 ++++++++++++
 2 files changed, 15 insertions(+)
 create mode 100644 .changelog/18269.txt

diff --git a/.changelog/18269.txt b/.changelog/18269.txt
new file mode 100644
index 00000000000..435b5b87549
--- /dev/null
+++ b/.changelog/18269.txt
@@ -0,0 +1,3 @@
+```release-note:bug
+client: Ignore stale server updates to prevent GCing allocations that should be running
+```
diff --git a/client/client.go b/client/client.go
index d5b353fe3ff..21a321dc26d 100644
--- a/client/client.go
+++ b/client/client.go
@@ -2171,6 +2171,18 @@ OUTER:
 		default:
 		}
 
+		// We have not received any new data, or received stale data. This may happen in
+		// an array of situations, the worst of which seems to be a blocking request
+		// timeout when the scheduler which we are contacting is newly added or recovering
+		// after a prolonged downtime.
+		//
+		// For full context, please see https://github.com/hashicorp/nomad/issues/18267
+		if resp.Index <= req.MinQueryIndex {
+			c.logger.Debug("Received stale allocation information. Retrying.",
+				"index", resp.Index, "min_index", req.MinQueryIndex)
+			continue OUTER
+		}
+
 		// Filter all allocations whose AllocModifyIndex was not incremented.
 		// These are the allocations who have either not been updated, or whose
 		// updates are a result of the client sending an update for the alloc.