From b5b445c1012c413bce47348b404c6d45c10a2713 Mon Sep 17 00:00:00 2001
From: Mahmood Ali <mahmood@hashicorp.com>
Date: Wed, 18 Sep 2019 08:09:13 -0400
Subject: [PATCH] add exponential backoff for docker api calls

---
 drivers/docker/driver.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go
index 59bc8c55a17..3367caeba8d 100644
--- a/drivers/docker/driver.go
+++ b/drivers/docker/driver.go
@@ -455,7 +455,7 @@ CREATE:
 
 		if attempted < 5 {
 			attempted++
-			time.Sleep(1 * time.Second)
+			time.Sleep(nextBackoff(attempted))
 			goto CREATE
 		}
 	} else if strings.Contains(strings.ToLower(createErr.Error()), "no such image") {
@@ -464,7 +464,7 @@ CREATE:
 		return nil, nstructs.NewRecoverableError(createErr, true)
 	} else if isDockerTransientError(createErr) && attempted < 5 {
 		attempted++
-		time.Sleep(1 * time.Second)
+		time.Sleep(nextBackoff(attempted))
 		goto CREATE
 	}
 
@@ -487,7 +487,7 @@ START:
 	if isDockerTransientError(startErr) {
 		if attempted < 5 {
 			attempted++
-			time.Sleep(1 * time.Second)
+			time.Sleep(nextBackoff(attempted))
 			goto START
 		}
 		return nstructs.NewRecoverableError(startErr, true)
@@ -496,6 +496,13 @@ START:
 	return recoverableErrTimeouts(startErr)
 }
 
+// nextBackoff returns appropriate docker backoff durations after attempted attempts.
+func nextBackoff(attempted int) time.Duration {
+	// attempts in 200ms, 800ms, 3.2s, 12.8s, 51.2s
+	// TODO: add randomization factor and extract to a helper
+	return 1 << (2 * uint64(attempted)) * 50 * time.Millisecond
+}
+
 // createImage creates a docker image either by pulling it from a registry or by
 // loading it from the file system
 func (d *Driver) createImage(task *drivers.TaskConfig, driverConfig *TaskConfig, client *docker.Client) (string, error) {