diff --git a/nomad/worker.go b/nomad/worker.go index 7d663479a20..f4ef1fb5cdb 100644 --- a/nomad/worker.go +++ b/nomad/worker.go @@ -27,6 +27,10 @@ const ( // the slower backoff backoffLimitSlow = 10 * time.Second + // backoffSchedulerVersionMismatch is the backoff between retries when the + // scheduler version mismatches that of the leader. + backoffSchedulerVersionMismatch = 30 * time.Second + // dequeueTimeout is used to timeout an evaluation dequeue so that // we can check if there is a shutdown event dequeueTimeout = 500 * time.Millisecond @@ -155,7 +159,16 @@ REQ: if time.Since(w.start) > dequeueErrGrace && !w.srv.IsShutdown() { w.logger.Printf("[ERR] worker: failed to dequeue evaluation: %v", err) } - if w.backoffErr(backoffBaselineSlow, backoffLimitSlow) { + + // Adjust the backoff based on the error. If it is a scheduler version + // mismatch we increase the baseline. + base, limit := backoffBaselineFast, backoffLimitSlow + if strings.Contains(err.Error(), "calling scheduler version") { + base = backoffSchedulerVersionMismatch + limit = backoffSchedulerVersionMismatch + } + + if w.backoffErr(base, limit) { return nil, "", true } goto REQ