Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Retry requests on a 429 that's a lock timeout #718

Merged
merged 1 commit into from
Aug 28, 2019
Merged

Retry requests on a 429 that's a lock timeout #718

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions lib/HttpClient/CurlClient.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ private function executeRequestWithRetries($opts, $absUrl)
$this->closeCurlHandle();
}

if ($this->shouldRetry($errno, $isPost, $rcode, $numRetries)) {
if ($this->shouldRetry($errno, $isPost, $rcode, $rbody, $numRetries)) {
$numRetries += 1;
$sleepSeconds = $this->sleepTime($numRetries);
usleep(intval($sleepSeconds * 1000000));
Expand Down Expand Up @@ -338,13 +338,16 @@ private function handleCurlError($url, $errno, $message, $numRetries)
* Checks if an error is a problem that we should retry on. This includes both
* socket errors that may represent an intermittent problem and some special
* HTTP statuses.
*
* @param int $errno
* @param bool $isPost
* @param int $rcode
* @param string $rbody
* @param int $numRetries
*
* @return bool
*/
private function shouldRetry($errno, $isPost, $rcode, $numRetries)
private function shouldRetry($errno, $isPost, $rcode, $rbody, $numRetries)
{
if ($numRetries >= Stripe::getMaxNetworkRetries()) {
return false;
Expand All @@ -367,6 +370,28 @@ private function shouldRetry($errno, $isPost, $rcode, $numRetries)
return true;
}

// 429 Too Many Requests
//
// There are a few different problems that can lead to a 429. The most
// common is rate limiting, on which we *don't* want to retry because
// that'd likely contribute to more contention problems. However, some
// 429s are lock timeouts, which is when a request conflicted with
// another request or an internal process on some particular object.
// These 429s are safe to retry.
if ($rcode === 429) {
// It's not great that we're doing this here. In a future version,
// we should decouple the retry logic from the CurlClient instance,
// so that we don't need to deserialize here (and also so that the
// retry logic applies to non-curl clients).
$resp = json_decode($rbody, true);
ob-stripe marked this conversation as resolved.
Show resolved Hide resolved
if ($resp !== null && array_key_exists('error', $resp)) {
$error = \Stripe\ErrorObject::constructFrom($resp['error']);
if ($error->code === \Stripe\ErrorObject::CODE_LOCK_TIMEOUT) {
return true;
}
}
}

// 500 Internal Server Error
//
// We only bother retrying these for non-POST requests. POSTs end up
Expand Down
43 changes: 35 additions & 8 deletions tests/Stripe/HttpClient/CurlClientTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ public function testShouldRetryOnTimeout()

$curlClient = new CurlClient();

$this->assertTrue($this->shouldRetryMethod->invoke($curlClient, CURLE_OPERATION_TIMEOUTED, true, 0, 0));
$this->assertTrue($this->shouldRetryMethod->invoke($curlClient, CURLE_OPERATION_TIMEOUTED, true, 0, "", 0));
}

public function testShouldRetryOnConnectionFailure()
Expand All @@ -139,7 +139,7 @@ public function testShouldRetryOnConnectionFailure()

$curlClient = new CurlClient();

$this->assertTrue($this->shouldRetryMethod->invoke($curlClient, CURLE_COULDNT_CONNECT, true, 0, 0));
$this->assertTrue($this->shouldRetryMethod->invoke($curlClient, CURLE_COULDNT_CONNECT, true, 0, "", 0));
}

public function testShouldRetryOnConflict()
Expand All @@ -148,7 +148,34 @@ public function testShouldRetryOnConflict()

$curlClient = new CurlClient();

$this->assertTrue($this->shouldRetryMethod->invoke($curlClient, 0, true, 409, 0));
$this->assertTrue($this->shouldRetryMethod->invoke($curlClient, 0, true, 409, "", 0));
}

public function testShouldRetryOn429WhenLockTimeout()
{
Stripe::setMaxNetworkRetries(2);

$curlClient = new CurlClient();

$this->assertTrue($this->shouldRetryMethod->invoke($curlClient, 0, true, 429, '{"error": {"code": "lock_timeout"}}', 0));
}

public function testShouldNotRetryOn429WhenNotLockTimeout()
{
Stripe::setMaxNetworkRetries(2);

$curlClient = new CurlClient();

$this->assertFalse($this->shouldRetryMethod->invoke($curlClient, 0, true, 429, '{"error": {"code": "rate_limited"}}', 0));
}

public function testShouldNotRetryOn429WhenInvalidJson()
{
Stripe::setMaxNetworkRetries(2);

$curlClient = new CurlClient();

$this->assertFalse($this->shouldRetryMethod->invoke($curlClient, 0, true, 429, 'this is not valid JSON', 0));
}

public function testShouldRetryOn500AndNonPost()
Expand All @@ -157,7 +184,7 @@ public function testShouldRetryOn500AndNonPost()

$curlClient = new CurlClient();

$this->assertTrue($this->shouldRetryMethod->invoke($curlClient, 0, false, 500, 0));
$this->assertTrue($this->shouldRetryMethod->invoke($curlClient, 0, false, 500, "", 0));
}

public function testShouldNotRetryOn500AndPost()
Expand All @@ -166,7 +193,7 @@ public function testShouldNotRetryOn500AndPost()

$curlClient = new CurlClient();

$this->assertFalse($this->shouldRetryMethod->invoke($curlClient, 0, true, 500, 0));
$this->assertFalse($this->shouldRetryMethod->invoke($curlClient, 0, true, 500, "", 0));
}

public function testShouldRetryOn503()
Expand All @@ -175,7 +202,7 @@ public function testShouldRetryOn503()

$curlClient = new CurlClient();

$this->assertTrue($this->shouldRetryMethod->invoke($curlClient, 0, true, 503, 0));
$this->assertTrue($this->shouldRetryMethod->invoke($curlClient, 0, true, 503, "", 0));
}

public function testShouldNotRetryAtMaximumCount()
Expand All @@ -184,7 +211,7 @@ public function testShouldNotRetryAtMaximumCount()

$curlClient = new CurlClient();

$this->assertFalse($this->shouldRetryMethod->invoke($curlClient, 0, true, 0, Stripe::getMaxNetworkRetries()));
$this->assertFalse($this->shouldRetryMethod->invoke($curlClient, 0, true, 0, "", Stripe::getMaxNetworkRetries()));
}

public function testShouldNotRetryOnCertValidationError()
Expand All @@ -193,7 +220,7 @@ public function testShouldNotRetryOnCertValidationError()

$curlClient = new CurlClient();

$this->assertFalse($this->shouldRetryMethod->invoke($curlClient, CURLE_SSL_PEER_CERTIFICATE, true, -1, 0));
$this->assertFalse($this->shouldRetryMethod->invoke($curlClient, CURLE_SSL_PEER_CERTIFICATE, true, -1, "", 0));
}

public function testSleepTimeShouldGrowExponentially()
Expand Down