From 23702f67129ff14e71e869a5a8677c1c57ca77cc Mon Sep 17 00:00:00 2001 From: Mark Hahnenberg Date: Thu, 2 May 2019 13:02:15 -0700 Subject: [PATCH] Use exponential backoff with jitter on 429 This can happen if there are too many clients trying to access the same base. To accomodate this, use exponential backoff instead of a fixed backoff. --- lib/internal_config.json | 3 ++- lib/run_action.js | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/internal_config.json b/lib/internal_config.json index 79c18a7f..ccb9a3a7 100644 --- a/lib/internal_config.json +++ b/lib/internal_config.json @@ -1,3 +1,4 @@ { - "RETRY_DELAY_IF_RATE_LIMITED": 5000 + "INITIAL_RETRY_DELAY_IF_RATE_LIMITED": 5000, + "MAX_RETRY_DELAY_IF_RATE_LIMITED": 300000 } diff --git a/lib/run_action.js b/lib/run_action.js index e431a057..417bcf37 100644 --- a/lib/run_action.js +++ b/lib/run_action.js @@ -6,7 +6,21 @@ var objectToQueryParamString = require('./object_to_query_param_string'); // This will become require('xhr') in the browser. var request = require('request'); +/* + * "Full Jitter" algorithm taken from https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + */ +function exponentialBackoffWithJitter(numberOfRetries, initialBackoffTimeMs, maxBackoffTimeMs) { + var rawBackoffTimeMs = initialBackoffTimeMs * Math.pow(2, numberOfRetries); + var clippedBackoffTimeMs = Math.min(maxBackoffTimeMs, rawBackoffTimeMs); + var jitteredBackoffTimeMs = Math.random() * clippedBackoffTimeMs; + return jitteredBackoffTimeMs; +} + function runAction(base, method, path, queryParams, bodyData, callback) { + return runActionImpl(base, method, path, queryParams, bodyData, callback, 0); +} + +function runActionImpl(base, method, path, queryParams, bodyData, callback, numAttempts) { var url = base._airtable._endpointUrl + '/v' + base._airtable._apiVersionMajor + '/' + base._id + path + '?' + objectToQueryParamString(queryParams); var headers = { @@ -49,9 +63,10 @@ function runAction(base, method, path, queryParams, bodyData, callback) { } if (resp.statusCode === 429 && !base._airtable._noRetryIfRateLimited) { + var backoffDelayMs = exponentialBackoffWithJitter(numAttempts, internalConfig.INITIAL_RETRY_DELAY_IF_RATE_LIMITED, internalConfig.MAX_RETRY_DELAY_IF_RATE_LIMITED); setTimeout(function() { - runAction(base, method, path, queryParams, bodyData, callback); - }, internalConfig.RETRY_DELAY_IF_RATE_LIMITED); + runAction(base, method, path, queryParams, bodyData, callback, numAttempts + 1); + }, backoffDelayMs); return; }