Use exponential backoff with jitter on 429

This can happen if there are too many clients trying to access the same base.
To accomodate this, use exponential backoff instead of a fixed backoff.

lib/internal_config.json

@@ -1,3 +1,4 @@
 {
-    "RETRY_DELAY_IF_RATE_LIMITED": 5000
+    "INITIAL_RETRY_DELAY_IF_RATE_LIMITED": 5000,
+    "MAX_RETRY_DELAY_IF_RATE_LIMITED": 300000
 }

lib/run_action.js

@@ -6,7 +6,21 @@ var objectToQueryParamString = require('./object_to_query_param_string');
 // This will become require('xhr') in the browser.
 var request = require('request');
 
+/*
+ * "Full Jitter" algorithm taken from https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
+ */
+function exponentialBackoffWithJitter(numberOfRetries, initialBackoffTimeMs, maxBackoffTimeMs) {
+    var rawBackoffTimeMs = initialBackoffTimeMs * Math.pow(2, numberOfRetries);
+    var clippedBackoffTimeMs = Math.min(maxBackoffTimeMs, rawBackoffTimeMs);
+    var jitteredBackoffTimeMs = Math.random() * clippedBackoffTimeMs;
+    return jitteredBackoffTimeMs;
+}
+
 function runAction(base, method, path, queryParams, bodyData, callback) {
+    return runActionImpl(base, method, path, queryParams, bodyData, callback, 0);
+}
+
+function runActionImpl(base, method, path, queryParams, bodyData, callback, numAttempts) {
     var url = base._airtable._endpointUrl + '/v' + base._airtable._apiVersionMajor + '/' + base._id + path + '?' + objectToQueryParamString(queryParams);
 
     var headers = {
@@ -49,9 +63,10 @@ function runAction(base, method, path, queryParams, bodyData, callback) {
         }
 
         if (resp.statusCode === 429 && !base._airtable._noRetryIfRateLimited) {
+            var backoffDelayMs = exponentialBackoffWithJitter(numAttempts, internalConfig.INITIAL_RETRY_DELAY_IF_RATE_LIMITED, internalConfig.MAX_RETRY_DELAY_IF_RATE_LIMITED);
             setTimeout(function() {
-                runAction(base, method, path, queryParams, bodyData, callback);
-            }, internalConfig.RETRY_DELAY_IF_RATE_LIMITED);
+                runAction(base, method, path, queryParams, bodyData, callback, numAttempts + 1);
+            }, backoffDelayMs);
             return;
         }