From a8211f24fcb975706e2075fd6d2367f6990543b5 Mon Sep 17 00:00:00 2001 From: Saransh Dhingra Date: Wed, 26 Apr 2023 00:31:06 +0530 Subject: [PATCH] feat(Storage): Add retry conformance tests (#5637) * feat(Storage): Add contextual retry capability. * feat(Storage): Add retry conformance tests. --- ...rage-emulator-retry-conformance-tests.yaml | 40 + Core/src/ExponentialBackoff.php | 33 +- Core/src/RequestWrapper.php | 34 +- Core/src/RestTrait.php | 4 +- Core/src/Upload/AbstractUploader.php | 4 +- Core/tests/Unit/ExponentialBackoffTest.php | 33 + Core/tests/Unit/RequestWrapperTest.php | 37 + .../Unit/Upload/ResumableUploaderTest.php | 43 + Storage/composer.json | 3 +- Storage/phpunit-conformance.xml.dist | 16 + Storage/src/Connection/Rest.php | 221 +++- Storage/src/Connection/RetryTrait.php | 234 ++++ Storage/src/StorageClient.php | 18 + Storage/src/StorageObject.php | 5 +- .../Conformance/RetryConformanceTest.php | 1032 +++++++++++++++++ .../tests/Conformance/data/retry_tests.json | 275 +++++ Storage/tests/Snippet/StorageObjectTest.php | 22 +- Storage/tests/Unit/Connection/RestTest.php | 57 +- .../tests/Unit/Connection/RetryTraitTest.php | 178 +++ Storage/tests/Unit/StorageObjectTest.php | 147 ++- 20 files changed, 2366 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/storage-emulator-retry-conformance-tests.yaml create mode 100644 Storage/phpunit-conformance.xml.dist create mode 100644 Storage/src/Connection/RetryTrait.php create mode 100644 Storage/tests/Conformance/RetryConformanceTest.php create mode 100644 Storage/tests/Conformance/data/retry_tests.json create mode 100644 Storage/tests/Unit/Connection/RetryTraitTest.php diff --git a/.github/workflows/storage-emulator-retry-conformance-tests.yaml b/.github/workflows/storage-emulator-retry-conformance-tests.yaml new file mode 100644 index 000000000000..6412d492dd8a --- /dev/null +++ b/.github/workflows/storage-emulator-retry-conformance-tests.yaml @@ -0,0 +1,40 @@ +on: + push: + branches: + - main + paths: + - 'Storage/**' + - '.github/workflows/storage-emulator-retry-conformance-tests.yaml' + pull_request: + paths: + - 'Storage/**' + - '.github/workflows/storage-emulator-retry-conformance-tests.yaml' +name: Run Storage Retry Conformance Tests With Emulator +jobs: + test: + runs-on: ubuntu-20.04 + + services: + emulator: + image: gcr.io/cloud-devrel-public-resources/storage-testbench:v0.35.0 + ports: + - 9000:9000 + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: '7.4' + + - name: Install dependencies + run: | + composer update --prefer-dist --no-interaction --no-suggest + + - name: Run storage retry conformance tests + run: | + vendor/bin/phpunit -c Storage/phpunit-conformance.xml.dist + env: + STORAGE_EMULATOR_HOST: http://localhost:9000 diff --git a/Core/src/ExponentialBackoff.php b/Core/src/ExponentialBackoff.php index 0f924d5f0699..ab3a4116eed3 100644 --- a/Core/src/ExponentialBackoff.php +++ b/Core/src/ExponentialBackoff.php @@ -44,14 +44,32 @@ class ExponentialBackoff */ private $calcDelayFunction; + /** + * @var callable|null + */ + private $retryListener; + /** * @param int $retries [optional] Number of retries for a failed request. - * @param callable $retryFunction [optional] returns bool for whether or not to retry + * @param callable $retryFunction [optional] returns bool for whether or not + * to retry + * @param callable $retryListener [optional] Runs after the + * $retryFunction. Unlike the $retryFunction,this function isn't + * responsible to decide if a retry should happen or not, but it gives the + * users flexibility to consume exception messages and add custom logic. + * Function definition should match: + * function (\Exception $e, int $attempt, array $arguments): array + * Ex: One might want to change headers on every retry, this function can + * be used to achieve such a functionality. */ - public function __construct($retries = null, callable $retryFunction = null) - { + public function __construct( + $retries = null, + callable $retryFunction = null, + callable $retryListener = null + ) { $this->retries = $retries !== null ? (int) $retries : 3; $this->retryFunction = $retryFunction; + $this->retryListener = $retryListener; // @todo revisit this approach // @codeCoverageIgnoreStart $this->delayFunction = static function ($delay) { @@ -74,7 +92,6 @@ public function execute(callable $function, array $arguments = []) $calcDelayFunction = $this->calcDelayFunction ?: [$this, 'calculateDelay']; $retryAttempt = 0; $exception = null; - while (true) { try { return call_user_func_array($function, $arguments); @@ -91,6 +108,14 @@ public function execute(callable $function, array $arguments = []) $delayFunction($calcDelayFunction($retryAttempt)); $retryAttempt++; + if ($this->retryListener) { + // Developer can modify the $arguments using the retryListener + // callback. + call_user_func_array( + $this->retryListener, + [$exception, $retryAttempt, &$arguments] + ); + } } } diff --git a/Core/src/RequestWrapper.php b/Core/src/RequestWrapper.php index 1f88f4dbd982..e80d6e1eeb5a 100644 --- a/Core/src/RequestWrapper.php +++ b/Core/src/RequestWrapper.php @@ -30,6 +30,7 @@ use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\StreamInterface; +use Google\ApiCore\AgentHeader; /** * The RequestWrapper is responsible for delivering and signing requests. @@ -173,6 +174,11 @@ public function __construct(array $config = []) * @type callable $restRetryFunction Sets the conditions for whether or * not a request should attempt to retry. Function signature should * match: `function (\Exception $ex) : bool`. + * @type callable $restRetryListener Runs after the restRetryFunction. + * This might be used to simply consume the exception and + * $arguments b/w retries. This returns the new $arguments thus + * allowing modification on demand for $arguments. For ex: + * changing the headers in b/w retries. * @type callable $restDelayFunction Executes a delay, defaults to * utilizing `usleep`. Function signature should match: * `function (int $delay) : void`. @@ -189,7 +195,8 @@ public function send(RequestInterface $request, array $options = []) $retryOptions = $this->getRetryOptions($options); $backoff = new ExponentialBackoff( $retryOptions['retries'], - $retryOptions['retryFunction'] + $retryOptions['retryFunction'], + $retryOptions['retryListener'], ); if ($retryOptions['delayFunction']) { @@ -202,7 +209,7 @@ public function send(RequestInterface $request, array $options = []) try { return $backoff->execute($this->httpHandler, [ - $this->applyHeaders($request), + $this->applyHeaders($request, $options), $this->getRequestOptions($options) ]); } catch (\Exception $ex) { @@ -252,7 +259,7 @@ public function sendAsync(RequestInterface $request, array $options = []) } return $asyncHttpHandler( - $this->applyHeaders($request), + $this->applyHeaders($request, $options), $this->getRequestOptions($options) )->then(null, function (\Exception $ex) use ($fn, $retryAttempt, $retryOptions) { $shouldRetry = $retryOptions['retryFunction']($ex, $retryAttempt); @@ -276,15 +283,29 @@ public function sendAsync(RequestInterface $request, array $options = []) * Applies headers to the request. * * @param RequestInterface $request A PSR-7 request. + * @param array $options * @return RequestInterface */ - private function applyHeaders(RequestInterface $request) + private function applyHeaders(RequestInterface $request, array $options = []) { $headers = [ 'User-Agent' => 'gcloud-php/' . $this->componentVersion, - 'x-goog-api-client' => 'gl-php/' . PHP_VERSION . ' gccl/' . $this->componentVersion, + AgentHeader::AGENT_HEADER_KEY => sprintf( + 'gl-php/%s gccl/%s', + PHP_VERSION, + $this->componentVersion + ), ]; + if (isset($options['retryHeaders'])) { + $headers[AgentHeader::AGENT_HEADER_KEY] = sprintf( + '%s %s', + $headers[AgentHeader::AGENT_HEADER_KEY], + implode(' ', $options['retryHeaders']) + ); + unset($options['retryHeaders']); + } + if ($this->shouldSignRequest) { $quotaProject = $this->quotaProject; $token = null; @@ -427,6 +448,9 @@ private function getRetryOptions(array $options) 'retryFunction' => isset($options['restRetryFunction']) ? $options['restRetryFunction'] : $this->retryFunction, + 'retryListener' => isset($options['restRetryListener']) + ? $options['restRetryListener'] + : null, 'delayFunction' => isset($options['restDelayFunction']) ? $options['restDelayFunction'] : $this->delayFunction, diff --git a/Core/src/RestTrait.php b/Core/src/RestTrait.php index 550f89b22c23..e2fa52b62333 100644 --- a/Core/src/RestTrait.php +++ b/Core/src/RestTrait.php @@ -90,8 +90,10 @@ public function send($resource, $method, array $options = [], $whitelisted = fal $requestOptions = $this->pluckArray([ 'restOptions', 'retries', + 'retryHeaders', 'requestTimeout', - 'restRetryFunction' + 'restRetryFunction', + 'restRetryListener', ], $options); try { diff --git a/Core/src/Upload/AbstractUploader.php b/Core/src/Upload/AbstractUploader.php index cc8692f6f1dd..cdd3b3095dfd 100644 --- a/Core/src/Upload/AbstractUploader.php +++ b/Core/src/Upload/AbstractUploader.php @@ -103,7 +103,9 @@ public function __construct( $this->requestOptions = array_intersect_key($options, [ 'restOptions' => null, 'retries' => null, - 'requestTimeout' => null + 'requestTimeout' => null, + 'restRetryFunction' => null, + 'restRetryListener' => null ]); $this->contentType = $options['contentType'] ?? 'application/octet-stream'; diff --git a/Core/tests/Unit/ExponentialBackoffTest.php b/Core/tests/Unit/ExponentialBackoffTest.php index 14d4c520ed84..bcf10449b4ba 100644 --- a/Core/tests/Unit/ExponentialBackoffTest.php +++ b/Core/tests/Unit/ExponentialBackoffTest.php @@ -160,4 +160,37 @@ public function delayProvider() [10, 60000000, 60000000] ]; } + + /** + * Tests whether `retryListener()` callback is + * properly invoked when exception occurs in the request being made. + */ + public function testRetryListener() + { + $args = ['foo' => 'bar']; + $retryListener = function ( + $ex, + $retryAttempt, + $arguments + ) { + self::assertEquals(0, $retryAttempt); + self::assertEquals('bar', $arguments[0]['foo']); + }; + + // Setting $retries to 0 so that retry doesn't happens after first + // failure. + $backoff = new ExponentialBackoff(0, null, $retryListener); + try { + $backoff->execute( + function () { + throw new \Exception('Intentionally failing request'); + }, + [$args] + ); + } catch (\Exception $err) { + // Do nothing. + // Catching the intentional failing call being made above: + // "Intentionally failing request" + } + } } diff --git a/Core/tests/Unit/RequestWrapperTest.php b/Core/tests/Unit/RequestWrapperTest.php index f7bd212bbb95..a2ca91384e40 100644 --- a/Core/tests/Unit/RequestWrapperTest.php +++ b/Core/tests/Unit/RequestWrapperTest.php @@ -631,6 +631,43 @@ public function testEmptyTokenThrowsException() $requestWrapper->send(new Request('GET', 'http://www.example.com')); } + + /** + * This test asserts that the retry related options and callbacks are + * properly mapped and set in the RequestWrapper's `$requestOptions` + * property. + */ + public function testPassingInRetryOptions() + { + $attempt = 0; + $retryFunctionCalled = false; + $retryListenerCalled = false; + $options = [ + 'restRetryFunction' => function () use (&$retryFunctionCalled) { + $retryFunctionCalled = true; + return true; + }, + 'restRetryListener' => function () use (&$retryListenerCalled) { + $retryListenerCalled = true; + }, + ]; + $request = new Request('GET', 'http://www.example.com'); + $requestWrapper = new RequestWrapper([ + 'authHttpHandler' => function () { + return new Response(200, [], '{"access_token": "abc"}'); + }, + 'httpHandler' => function () use (&$attempt) { + if ($attempt++ < 1) { + throw new \Exception('retry!'); + } + return new Response(200, []); + } + ]); + $requestWrapper->send($request, $options); + + $this->assertTrue($retryFunctionCalled); + $this->assertTrue($retryListenerCalled); + } } //@codingStandardsIgnoreStart diff --git a/Core/tests/Unit/Upload/ResumableUploaderTest.php b/Core/tests/Unit/Upload/ResumableUploaderTest.php index 1df52d036a3b..7b0a5251a5b7 100644 --- a/Core/tests/Unit/Upload/ResumableUploaderTest.php +++ b/Core/tests/Unit/Upload/ResumableUploaderTest.php @@ -174,6 +174,49 @@ public function testResumeFinishedUpload() ); } + /** + * This tests whether retry related options are properly set in the + * abstract uploader class. Since we already had these tests for + * ResumableUploader class which extends the AbstractUploader class, + * thus testing it here would be sufficient. + */ + public function testRetryOptionsPassing() + { + $attempt = 0; + $retryFunctionCalled = false; + $retryListenerCalled = false; + $requestWrapper = new RequestWrapper([ + 'httpHandler' => function () use (&$attempt) { + if ($attempt++ < 1) { + throw new \Exception('retry!'); + } + return new Response(200, [], $this->successBody); + }, + 'authHttpHandler' => function () { + return new Response(200, [], '{"access_token": "abc"}'); + }, + ]); + $options = [ + 'restRetryFunction' => function () use (&$retryFunctionCalled) { + $retryFunctionCalled = true; + return true; + }, + 'restRetryListener' => function () use (&$retryListenerCalled) { + $retryListenerCalled = true; + }, + ]; + $uploader = new ResumableUploader( + $requestWrapper, + $this->stream, + 'http://www.example.com', + $options + ); + $uploader->upload(); + + $this->assertTrue($retryFunctionCalled); + $this->assertTrue($retryListenerCalled); + } + public function testThrowsExceptionWhenAttemptsAsyncUpload() { $this->expectException(GoogleException::class); diff --git a/Storage/composer.json b/Storage/composer.json index 46804d48e40b..e50c070b77da 100644 --- a/Storage/composer.json +++ b/Storage/composer.json @@ -6,7 +6,8 @@ "require": { "php": ">=7.4", "google/cloud-core": "^1.43", - "google/crc32": "^0.2.0" + "google/crc32": "^0.2.0", + "ramsey/uuid": "^4.2.3" }, "require-dev": { "phpunit/phpunit": "^9.0", diff --git a/Storage/phpunit-conformance.xml.dist b/Storage/phpunit-conformance.xml.dist new file mode 100644 index 000000000000..ecae94837c3b --- /dev/null +++ b/Storage/phpunit-conformance.xml.dist @@ -0,0 +1,16 @@ + + + + + src + + + src/V[!a-zA-Z]* + + + + + tests/Conformance + + + diff --git a/Storage/src/Connection/Rest.php b/Storage/src/Connection/Rest.php index ef6ef7db8cf4..9ec98d60d6f2 100644 --- a/Storage/src/Connection/Rest.php +++ b/Storage/src/Connection/Rest.php @@ -17,9 +17,11 @@ namespace Google\Cloud\Storage\Connection; +use Google\ApiCore\AgentHeader; use Google\Cloud\Core\RequestBuilder; use Google\Cloud\Core\RequestWrapper; use Google\Cloud\Core\RestTrait; +use Google\Cloud\Storage\Connection\RetryTrait; use Google\Cloud\Core\Upload\AbstractUploader; use Google\Cloud\Core\Upload\MultipartUploader; use Google\Cloud\Core\Upload\ResumableUploader; @@ -29,11 +31,14 @@ use Google\Cloud\Storage\StorageClient; use Google\CRC32\Builtin; use Google\CRC32\CRC32; +use GuzzleHttp\Exception\RequestException; use GuzzleHttp\Psr7\MimeType; use GuzzleHttp\Psr7\Request; use GuzzleHttp\Psr7\Utils; +use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\StreamInterface; +use Ramsey\Uuid\Uuid; /** * Implementation of the @@ -41,9 +46,19 @@ */ class Rest implements ConnectionInterface { - use RestTrait; + use RestTrait { + send as private traitSend; + } + use RetryTrait; use UriTrait; + /** + * Header and value that helps us identify a transcoded obj + * w/o making a metadata(info) call. + */ + private const TRANSCODED_OBJ_HEADER_KEY = 'X-Goog-Stored-Content-Encoding'; + private const TRANSCODED_OBJ_HEADER_VAL = 'gzip'; + /** * @deprecated */ @@ -75,6 +90,12 @@ class Rest implements ConnectionInterface */ private $apiEndpoint; + /** + * @var callable + * value null accepted + */ + private $restRetryFunction; + /** * @param array $config */ @@ -98,6 +119,7 @@ public function __construct(array $config = []) )); $this->projectId = $this->pluck('projectId', $config, false); + $this->restRetryFunction = (isset($config['restRetryFunction'])) ? $config['restRetryFunction'] : null; } /** @@ -249,12 +271,68 @@ public function patchObject(array $args = []) */ public function downloadObject(array $args = []) { + // This makes sure we honour the range headers specified by the user + $requestedBytes = $this->getRequestedBytes($args); + $resultStream = Utils::streamFor(null); + $transcodedObj = false; + list($request, $requestOptions) = $this->buildDownloadObjectParams($args); - return $this->requestWrapper->send( + $invocationId = Uuid::uuid4()->toString(); + $requestOptions['retryHeaders'] = self::getRetryHeaders($invocationId, 1); + $requestOptions['restRetryFunction'] = $this->getRestRetryFunction('objects', 'get', $requestOptions); + // We try to deduce if the object is a transcoded object when we receive the headers. + $requestOptions['restOptions']['on_headers'] = function ($response) use (&$transcodedObj) { + $header = $response->getHeader(self::TRANSCODED_OBJ_HEADER_KEY); + if (is_array($header) && in_array(self::TRANSCODED_OBJ_HEADER_VAL, $header)) { + $transcodedObj = true; + } + }; + $requestOptions['restRetryListener'] = function ( + \Exception $e, + $retryAttempt, + &$arguments + ) use ( + $resultStream, + $requestedBytes, + $invocationId + ) { + // if the exception has a response for us to use + if ($e instanceof RequestException && $e->hasResponse()) { + $msg = (string) $e->getResponse()->getBody(); + + $fetchedStream = Utils::streamFor($msg); + + // add the partial response to our stream that we will return + Utils::copyToStream($fetchedStream, $resultStream); + + // Start from the byte that was last fetched + $startByte = intval($requestedBytes['startByte']) + $resultStream->getSize(); + $endByte = $requestedBytes['endByte']; + + // modify the range headers to fetch the remaining data + $arguments[1]['headers']['Range'] = sprintf('bytes=%s-%s', $startByte, $endByte); + $arguments[0] = $this->modifyRequestForRetry($arguments[0], $retryAttempt, $invocationId); + } + }; + + $fetchedStream = $this->requestWrapper->send( $request, $requestOptions )->getBody(); + + // If our object is a transcoded object, then Range headers are not honoured. + // That means even if we had a partial download available, the final obj + // that was fetched will contain the complete object. So, we don't need to copy + // the partial stream, we can just return the stream we fetched. + if ($transcodedObj) { + return $fetchedStream; + } + + Utils::copyToStream($fetchedStream, $resultStream); + + $resultStream->seek(0); + return $resultStream; } /** @@ -302,6 +380,15 @@ public function insertObject(array $args = []) ] ]; + // Passing the preconditions we want to extract out of arguments + // into our query params. + $preconditions = self::$condIdempotentOps['objects.insert']; + foreach ($preconditions as $precondition) { + if (isset($args[$precondition])) { + $uriParams['query'][$precondition] = $args[$precondition]; + } + } + return new $uploaderClass( $this->requestWrapper, $args['data'], @@ -355,12 +442,26 @@ private function resolveUploadOptions(array $args) 'chunkSize', 'contentType', 'metadata', - 'uploadProgressCallback' + 'uploadProgressCallback', + 'restDelayFunction', + 'restCalcDelayFunction', ]; $args['uploaderOptions'] = array_intersect_key($args, array_flip($uploaderOptionKeys)); $args = array_diff_key($args, array_flip($uploaderOptionKeys)); + // Passing on custom retry function to $args['uploaderOptions'] + $retryFunc = $this->getRestRetryFunction( + 'objects', + 'insert', + $args + ); + $args['uploaderOptions']['restRetryFunction'] = $retryFunc; + + $args['uploaderOptions'] = $this->addRetryHeaderLogic( + $args['uploaderOptions'] + ); + return $args; } @@ -599,4 +700,118 @@ protected function supportsBuiltinCrc32c() { return Builtin::supports(CRC32::CASTAGNOLI); } + + /** + * Add the required retry function and send the request. + * + * @param string $resource resource name, eg: buckets. + * @param string $method method name, eg: get + * @param array $options [optional] Options used to build out the request. + * @param array $whitelisted [optional] + */ + public function send($resource, $method, array $options = [], $whitelisted = false) + { + $retryMap = [ + 'projects.resources.serviceAccount' => 'serviceaccount', + 'projects.resources.hmacKeys' => 'hmacKey', + 'bucketAccessControls' => 'bucket_acl', + 'defaultObjectAccessControls' => 'default_object_acl', + 'objectAccessControls' => 'object_acl' + ]; + $retryResource = isset($retryMap[$resource]) ? $retryMap[$resource] : $resource; + $options['restRetryFunction'] = $this->restRetryFunction ?? $this->getRestRetryFunction( + $retryResource, + $method, + $options + ); + + $options = $this->addRetryHeaderLogic($options); + + return $this->traitSend($resource, $method, $options); + } + + /** + * Adds the retry headers to $args which amends retry hash and attempt + * count to the required header. + * @param array $args + * @return array + */ + private function addRetryHeaderLogic(array $args) + { + $invocationId = Uuid::uuid4()->toString(); + $args['retryHeaders'] = self::getRetryHeaders($invocationId, 1); + + // Adding callback logic to update headers while retrying + $args['restRetryListener'] = function ( + \Exception $e, + $retryAttempt, + &$arguments + ) use ( + $invocationId + ) { + $arguments[0] = $this->modifyRequestForRetry( + $arguments[0], + $retryAttempt, + $invocationId + ); + }; + + return $args; + } + + private function modifyRequestForRetry( + RequestInterface $request, + int $retryAttempt, + string $invocationId + ) { + $changes = self::getRetryHeaders($invocationId, $retryAttempt + 1); + $headerLine = $request->getHeaderLine(AgentHeader::AGENT_HEADER_KEY); + + // An associative array to contain final header values as + // $headerValueKey => $headerValue + $headerElements = []; + + // Adding existing values + $headerLineValues = explode(' ', $headerLine); + foreach ($headerLineValues as $value) { + $key = explode('/', $value)[0]; + $headerElements[$key] = $value; + } + + // Adding changes with replacing value if $key already present + foreach ($changes as $change) { + $key = explode('/', $change)[0]; + $headerElements[$key] = $change; + } + + return $request->withHeader( + AgentHeader::AGENT_HEADER_KEY, + implode(' ', $headerElements) + ); + } + + /** + * Util function to compute the bytes requested for a download request. + * + * @param array $options Request options + * @return array + */ + private function getRequestedBytes(array $options) + { + $startByte = 0; + $endByte = ''; + + if (isset($options['restOptions']) && isset($options['restOptions']['headers'])) { + $headers = $options['restOptions']['headers']; + if (isset($headers['Range']) || isset($headers['range'])) { + $header = isset($headers['Range']) ? $headers['Range'] : $headers['range']; + $range = explode('=', $header); + $bytes = explode('-', $range[1]); + $startByte = $bytes[0]; + $endByte = $bytes[1]; + } + } + + return compact('startByte', 'endByte'); + } } diff --git a/Storage/src/Connection/RetryTrait.php b/Storage/src/Connection/RetryTrait.php new file mode 100644 index 000000000000..253a10b682d4 --- /dev/null +++ b/Storage/src/Connection/RetryTrait.php @@ -0,0 +1,234 @@ + ['ifMetagenerationMatch', 'etag'], + // Currently etag is not supported, so this preCondition never available + 'buckets.setIamPolicy' => ['etag'], + 'buckets.update' => ['ifMetagenerationMatch', 'etag'], + 'hmacKey.update' => ['etag'], + 'objects.compose' => ['ifGenerationMatch'], + 'objects.copy' => ['ifGenerationMatch'], + 'objects.delete' => ['ifGenerationMatch'], + 'objects.insert' => ['ifGenerationMatch', 'ifGenerationNotMatch'], + 'objects.patch' => ['ifMetagenerationMatch', 'etag'], + 'objects.rewrite' => ['ifGenerationMatch'], + 'objects.update' => ['ifMetagenerationMatch'] + ]; + + /** + * Retry strategies which enforce certain behaviour like: + * - Always retrying a call when an exception occurs(within the limits of 'max retries'). + * - Never retrying a call when an exception occurs. + * - Retrying only when the operation is considered idempotent(default). + * These configurations are supplied for per api call basis. + * + */ + + /** + * Header that identifies a specific request hash. The + * hash needs to stay the same for multiple retries. + */ + private static $INVOCATION_ID_HEADER = 'gccl-invocation-id'; + + /** + * Header that identifies the attempt count for a request. The + * value will increment by 1 with every retry. + */ + private static $ATTEMPT_COUNT_HEADER = 'gccl-attempt-count'; + + /** + * Return a retry decider function. + * + * @param string $resource resource name, eg: buckets. + * @param string $method method name, eg: get + * @param array $args + * @return callable + */ + private function getRestRetryFunction($resource, $method, array $args) + { + if (isset($args['restRetryFunction'])) { + return $args['restRetryFunction']; + } + $methodName = sprintf('%s.%s', $resource, $method); + $isOpIdempotent = in_array($methodName, self::$idempotentOps); + $preconditionNeeded = array_key_exists($methodName, self::$condIdempotentOps); + $preconditionSupplied = $this->isPreConditionSupplied($methodName, $args); + $retryStrategy = isset($args['retryStrategy']) ? + $args['retryStrategy'] : + StorageClient::RETRY_IDEMPOTENT; + + return function ( + \Exception $exception + ) use ( + $isOpIdempotent, + $preconditionNeeded, + $preconditionSupplied, + $retryStrategy + ) { + return $this->retryDeciderFunction( + $exception, + $isOpIdempotent, + $preconditionNeeded, + $preconditionSupplied, + $retryStrategy + ); + }; + } + + /** + * This function returns true when the user given + * precondtions ($preConditions) has values that are present + * in the precondition map ($this->condIdempotentMap) for that method. + * eg: condIdempotentMap has entry 'objects.copy' => ['ifGenerationMatch'], + * if the user has given 'ifGenerationMatch' in the 'objects.copy' operation, + * it will be available in the $preConditions + * as an array ['ifGenerationMatch']. This makes the array_intersect + * function return a non empty result and this function returns true. + * + * @param string $methodName method name, eg: buckets.get. + * @param array $args arguments which include preconditions provided, + * eg: ['ifGenerationMatch' => 0]. + * @return bool + */ + private function isPreConditionSupplied($methodName, array $args) + { + if (isset(self::$condIdempotentOps[$methodName])) { + // return true if required precondition are given. + return !empty(array_intersect( + self::$condIdempotentOps[$methodName], + array_keys($args) + )); + } + return false; + } + + /** + * Decide whether the op needs to be retried or not. + * + * @param \Exception $exception The exception object received + * while sending the request. + * @param int $currentAttempt Current retry attempt. + * @param bool $isIdempotent + * @param bool $preconditionNeeded + * @param bool $preconditionSupplied + * @param int $maxRetries + * @return bool + */ + private function retryDeciderFunction( + \Exception $exception, + $isIdempotent, + $preconditionNeeded, + $preconditionSupplied, + $retryStrategy + ) { + if ($retryStrategy == StorageClient::RETRY_NEVER) { + return false; + } + + $statusCode = $exception->getCode(); + // Retry if the exception status code matches + // with one of the retriable status code and + // the operation is either idempotent or conditionally + // idempotent with preconditions supplied. + + if (in_array($statusCode, self::$httpRetryCodes)) { + if ($retryStrategy == StorageClient::RETRY_ALWAYS) { + return true; + } elseif ($isIdempotent) { + return true; + } elseif ($preconditionNeeded) { + return $preconditionSupplied; + } + } + + return false; + } + + /** + * Utility func that returns the list of headers that need to be + * attached to every request and its retries. + */ + private static function getRetryHeaders($invocationId, $attemptCount) + { + return [ + sprintf('%s/%s', self::$INVOCATION_ID_HEADER, $invocationId), + sprintf('%s/%d', self::$ATTEMPT_COUNT_HEADER, $attemptCount) + ]; + } +} diff --git a/Storage/src/StorageClient.php b/Storage/src/StorageClient.php index 4b396ea05969..e2aceb8ab683 100644 --- a/Storage/src/StorageClient.php +++ b/Storage/src/StorageClient.php @@ -53,6 +53,24 @@ class StorageClient const READ_ONLY_SCOPE = 'https://www.googleapis.com/auth/devstorage.read_only'; const READ_WRITE_SCOPE = 'https://www.googleapis.com/auth/devstorage.read_write'; + /** + * Retry strategy to signify that we never want to retry an operation + * even if the error is retryable. + * + * We can set $options['retryStrategy'] to one of "always", "never" and + * "idempotent". + */ + const RETRY_NEVER = 'never'; + /** + * Retry strategy to signify that we always want to retry an operation. + */ + const RETRY_ALWAYS = 'always'; + /** + * This is the default. This signifies that we want to retry an operation + * only if it is retryable and the error is retryable. + */ + const RETRY_IDEMPOTENT = 'idempotent'; + /** * @var ConnectionInterface Represents a connection to Storage. */ diff --git a/Storage/src/StorageObject.php b/Storage/src/StorageObject.php index 14089f603156..c910aa2e6f0d 100644 --- a/Storage/src/StorageObject.php +++ b/Storage/src/StorageObject.php @@ -621,7 +621,10 @@ public function downloadToFile($path, array $options = []) } /** - * Download an object as a stream. + * Download an object as a stream. The library will attempt to resume the download + * if a retry-able error is thrown. An attempt to fetch the remaining file will + * be made only if the user has not supplied a custom retry + * function of their own. * * Please note Google Cloud Storage respects the Range header as specified * by [RFC7233](https://tools.ietf.org/html/rfc7233#section-3.1). See below diff --git a/Storage/tests/Conformance/RetryConformanceTest.php b/Storage/tests/Conformance/RetryConformanceTest.php new file mode 100644 index 000000000000..818778a3584b --- /dev/null +++ b/Storage/tests/Conformance/RetryConformanceTest.php @@ -0,0 +1,1032 @@ + self::$emulatorUrl + ]); + self::$storageClient = new StorageClient([ + 'apiEndpoint' => self::$emulatorUrl, + 'projectId' => self::$projectId, + 'credentialsFetcher' => new InsecureCredentials() + ]); + + $data = json_decode(file_get_contents(__DIR__ . '/data/retry_tests.json'), true); + $scenarios = $data['retryTests']; + $methodInvocations = self::getMethodInvocationMapping(); + + // create the permutations to be used for tests + foreach ($scenarios as $scenario) { + $scenarioId = $scenario['id']; + $errorCases = $scenario['cases']; + $methods = $scenario['methods']; + $expectedSuccess = $scenario['expectSuccess']; + $preconditionProvided = $scenario['preconditionProvided']; + + if (!isset(self::$cases[$scenarioId])) { + self::$cases[$scenarioId] = []; + } + + foreach ($errorCases as $row) { + $instructions = $row['instructions']; + foreach ($methods as $method) { + $methodName = $method['name']; + $methodGroup = isset($method['group']) ? $method['group'] : null; + $resources = $method['resources']; + + if (array_key_exists($methodName, $methodInvocations)) { + foreach ($methodInvocations[$methodName] as $invocationIndex => $callable) { + self::$cases[$scenarioId][] = compact( + 'methodName', + 'methodGroup', + 'instructions', + 'resources', + 'expectedSuccess', + 'preconditionProvided', + 'invocationIndex' + ); + } + } + } + } + } + } + + public function casesProvider() + { + self::set_up_before_class(); + // These scenario IDs will be run. + // Omit certain IDs for debugging or testing only + // certain cases. + $scenarios = [1, 2, 3, 4, 5, 6, 7, 8]; + + $cases = []; + + foreach ($scenarios as $scenarioId) { + $cases = array_merge($cases, self::$cases[$scenarioId]); + } + + return $cases; + } + + /** + * This tests that when we supply a header range with a start byte + * set, then the resulting stream is of the expected size. + * The same is tested when the download is interrupted and resumed. + */ + public function testDownloadAsStreamForStartBytesGiven() + { + $bucketName = uniqid(self::$bucketPrefix); + $objectName = uniqid(self::$objectPrefix); + $content = random_bytes(512 * 1024); + $bucket = self::$storageClient->createBucket($bucketName); + $object = $bucket->upload($content, ['name' => $objectName]); + + $options = [ + 'restOptions' => [ + 'headers' => [ + 'Range' => sprintf('bytes=%s-', 200 * 1024) + ] + ] + ]; + + $stream = $object->downloadAsStream($options); + // since the object is 512K, and our Range header + // requests start from 200K + $this->assertEquals(312 * 1024, $stream->getSize()); + + // Now lets test the same when the download is interrupted + $caseId = $this->createRetryTestResource("storage.objects.get", ["return-broken-stream-after-256K"]); + $options = [ + 'restOptions' => [ + 'headers' => [ + 'Range' => sprintf('bytes=%s-', 200 * 1024), + 'x-retry-test-id' => $caseId + ] + ] + ]; + $stream = $object->downloadAsStream($options); + $this->assertEquals(312 * 1024, $stream->getSize()); + + // Make sure the test case was used + $this->assertTrue($this->checkCaseCompletion($caseId)); + + // cleanup + $object->delete(); + $bucket->delete(); + } + + /** + * When the download is interrupted for a transcoded object, + * the retry sends the whole object and the bytes header is not respected. + * @see: https://cloud.google.com/storage/docs/transcoding + */ + public function testDownloadAsStreamForTranscodedObj() + { + $bucketName = uniqid(self::$bucketPrefix); + $objectName = uniqid(self::$objectPrefix).".txt.gz"; + $content = gzencode((string)random_bytes(512 * 1024)); + $bucket = self::$storageClient->createBucket($bucketName); + $object = $bucket->upload($content, [ + 'name' => $objectName, + 'metadata' => ['contentType' => 'text/plain','contentEncoding' => 'gzip'] + ]); + + $options = [ + 'restOptions' => [ + 'headers' => [ + 'Range' => sprintf('bytes=%s-', 200 * 1024) + ] + ] + ]; + + $stream = $object->downloadAsStream($options); + // Even though our Range header specifies the starting + // byte of 200K, but because the object is transcoded + // the full object is returned + $this->assertEquals(512 * 1024, $stream->getSize()); + + // Now we test the same for an interrupted download + $caseId = $this->createRetryTestResource("storage.objects.get", ["return-broken-stream-after-256K"]); + $options = [ + 'restOptions' => [ + 'headers' => [ + 'Range' => sprintf('bytes=%s-', 200 * 1024), + 'x-retry-test-id' => $caseId + ] + ] + ]; + $stream = $object->downloadAsStream($options); + $this->assertEquals(512 * 1024, $stream->getSize()); + + // cleanup + $object->delete(); + $bucket->delete(); + } + + + /** + * @dataProvider casesProvider + * All the retry conformance cases are tested here. + */ + public function testOps( + $methodName, + $methodGroup, + $instructions, + $resources, + $expectedSuccess, + $precondtionProvided, + $invocationIndex + ) { + $caseId = $this->createRetryTestResource($methodName, $instructions, null); + + $methodInvocations = self::getMethodInvocationMapping(); + $callable = $methodInvocations[$methodName][$invocationIndex]; + + if (!$expectedSuccess) { + $this->expectException('Exception'); + } + + $options = [ + 'restOptions' => [ + 'headers' => [ + 'x-retry-test-id' => $caseId + ] + ] + ]; + + // create the resources needed for test to run + $resourceIds = self::createResources(array_flip($resources), $methodGroup); + + // call the implementation + try { + call_user_func($callable, $resourceIds, $options, $precondtionProvided, $methodGroup); + // if an exception was thrown, then this block would never reach + if ($expectedSuccess) { + $this->assertTrue(true); + } + } catch (SkippedTestError $e) { + // Don't treat a skipped test as an exception. + // For example we skip tests when the only precondition is Etags. + $this->markTestSkipped($e->getMessage()); + } + + self::disposeResources($resourceIds); + + if (!$this->checkCaseCompletion($caseId)) { + $this->fail(sprintf( + 'The test case didn\'t complete for %s(invocation: %d).', + $methodName, + $invocationIndex + )); + } + } + + /** + * Create a Retry Test Resource by sending a request to the testbench emulator. + * @return string + */ + private function createRetryTestResource(string $method, array $instruction) + { + $data = [ + 'json' => ["instructions" => [ + $method => $instruction + ]] + ]; + $response = self::$httpClient->request('POST', 'retry_test', $data); + + $responseObj = json_decode($response->getBody()->getContents()); + return $responseObj->id; + } + + /** + * Helper method that checks if a test case resource has been completed or not. + * + * @param string $caseId The test case resource ID + * @return boolean + */ + private function checkCaseCompletion(string $caseId) + { + $response = self::$httpClient->request('GET', sprintf('retry_test/%s', $caseId)); + $obj = json_decode($response->getBody()->getContents()); + + return $obj->completed; + } + + /** + * Lists the different ways of invocing an API. + */ + private static function getMethodInvocationMapping() + { + return [ + 'storage.bucket_acl.get' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + + $bucket = self::$storageClient->bucket($bucketName); + $acl = $bucket->acl(); + + // this makes the storage.bucket_acl.get call + $options['entity'] = 'allUsers'; + $acl->get($options); + }, + ], + 'storage.bucket_acl.list' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + + $bucket = self::$storageClient->bucket($bucketName); + $acl = $bucket->acl(); + + // this makes the storage.bucket_acl.list call + $acl->get($options); + }, + ], + 'storage.buckets.delete' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + + $bucket = self::$storageClient->bucket($bucketName); + $bucket->delete($options); + }, + ], + 'storage.buckets.get' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + + $bucket = self::$storageClient->bucket($bucketName); + $bucket->reload($options); + }, + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + + $bucket = self::$storageClient->bucket($bucketName); + $exists = $bucket->exists($options); + }, + ], + 'storage.buckets.getIamPolicy' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + + $bucket = self::$storageClient->bucket($bucketName); + $iam = $bucket->iam(); + $iam->reload($options); + } + ], + 'storage.buckets.insert' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = uniqid(self::$bucketPrefix); + $bucket = self::$storageClient->createBucket($bucketName, $options); + $name = $bucket->name(); + + $bucket->delete(); + }, + ], + 'storage.buckets.list' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + + $buckets = self::$storageClient->buckets($options); + + // added this to trigger the API call + foreach ($buckets as $bucket) { + } + }, + ], + 'storage.buckets.lockRetentionPolicy' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $bucket = self::$storageClient->bucket($bucketName); + + $metageneration = $bucket->info()['metageneration']; + $options['IfMetagenerationMatch'] = $metageneration; + $bucket->lockRetentionPolicy($options); + } + ], + 'storage.buckets.testIamPermissions' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + + $bucket = self::$storageClient->bucket($bucketName); + $iam = $bucket->iam(); + $iam->testPermissions([], $options); + } + ], + 'storage.default_object_acl.get' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + + $bucket = self::$storageClient->bucket($bucketName); + $acl = $bucket->defaultAcl(); + + $options['entity'] = 'allUsers'; + $acl->get($options); + } + ], + 'storage.default_object_acl.list' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + + $bucket = self::$storageClient->bucket($bucketName); + $acl = $bucket->defaultAcl(); + + $acl->get($options); + } + ], + 'storage.hmacKey.delete' => [ + function ($resourceIds, $options, $precondition = false) { + $accessId = $resourceIds['hmacKeyId']; + + $key = self::$storageClient->hmacKey($accessId); + $key->update('INACTIVE'); + $key->delete($options); + } + ], + 'storage.hmacKey.get' => [ + function ($resourceIds, $options, $precondition = false) { + $accessId = $resourceIds['hmacKeyId']; + + $key = self::$storageClient->hmacKey($accessId); + $key->reload($options); + } + ], + 'storage.hmacKey.list' => [ + function ($resourceIds, $options, $precondition = false) { + $keys = self::$storageClient->hmacKeys($options); + + // Added this to trigger the API call + foreach ($keys as $key) { + } + } + ], + 'storage.notifications.delete' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $notificationId = $resourceIds['notificationId']; + + $bucket = self::$storageClient->bucket($bucketName); + $notification = $bucket->notification($notificationId); + $notification->delete($options); + } + ], + 'storage.notifications.get' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $notificationId = $resourceIds['notificationId']; + + $bucket = self::$storageClient->bucket($bucketName); + $notification = $bucket->notification($notificationId); + $notification->reload($options); + }, + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $notificationId = $resourceIds['notificationId']; + + $bucket = self::$storageClient->bucket($bucketName); + $notification = $bucket->notification($notificationId); + $notification->exists($options); + } + ], + 'storage.notifications.list' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $notificationId = $resourceIds['notificationId']; + + $bucket = self::$storageClient->bucket($bucketName); + $notifs = $bucket->notifications($options); + // Added this to trigger the API call + foreach ($notifs as $notif) { + } + } + ], + 'storage.object_acl.get' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + $acl = $object->acl(); + $options['entity'] = 'allUsers'; + if ($precondition) { + $options['ifGenerationMatch'] = $object->info()['generation']; + } + $acl->get($options); + } + ], + 'storage.object_acl.list' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + $acl = $object->acl(); + if ($precondition) { + $options['ifGenerationMatch'] = $object->info()['generation']; + } + $acl->get($options); + } + ], + 'storage.objects.get' => [ + function ($resourceIds, $options, $precondition = false, $methodGroup = null) { + if (!is_null($methodGroup)) { + self::markTestSkipped("Test only needs to run for resumable downloads"); + } + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + $object->reload($options); + }, + function ($resourceIds, $options, $precondition = false, $methodGroup = null) { + if (!is_null($methodGroup)) { + self::markTestSkipped("Test only needs to run for resumable downloads"); + } + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + $object->exists($options); + }, + function ($resourceIds, $options, $precondition = false, $methodGroup = null) { + if ($methodGroup !== 'storage.objects.download') { + self::markTestSkipped("Test only needs to run for getObject"); + } + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + $object->downloadAsStream($options); + }, + function ($resourceIds, $options, $precondition = false, $methodGroup = null) { + if ($methodGroup !== 'storage.objects.download') { + self::markTestSkipped("Test only needs to run for getObject"); + } + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + $object->downloadAsString($options); + }, + function ($resourceIds, $options, $precondition = false, $methodGroup = null) { + if ($methodGroup !== 'storage.objects.download') { + self::markTestSkipped("Test only needs to run for getObject"); + } + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + $object->downloadToFile('php://temp', $options); + } + ], + 'storage.objects.list' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + + $bucket = self::$storageClient->bucket($bucketName); + $objects = $bucket->objects($options); + // Added this to trigger the API call + foreach ($objects as $obj) { + } + } + ], + 'storage.serviceaccount.get' => [ + function ($resourceIds, $options, $precondition = false) { + self::$storageClient->getServiceAccount($options); + } + ], + 'storage.buckets.patch' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + + $options['labels'] = ['key' => 'value']; + $bucket = self::$storageClient->bucket($bucketName); + + if ($precondition) { + $options['ifMetagenerationMatch'] = $bucket->info()['metageneration']; + } + + $bucket->update($options); + } + ], + 'storage.buckets.setIamPolicy' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + + $bucket = self::$storageClient->bucket($bucketName); + $iam = $bucket->iam(); + $policy = $iam->policy(); + + $policy['bindings'][0]['members'] = ['user:test@test.com']; + $iam->setPolicy($policy, $options); + + if ($precondition) { + self::markTestSkipped('Etag is currently not supported.'); + } + + $bucket->update($options); + } + ], + 'storage.buckets.update' => [ + // This isn't used in the library + ], + 'storage.hmacKey.update' => [ + function ($resourceIds, $options, $precondition) { + $accessId = $resourceIds['hmacKeyId']; + + $key = self::$storageClient->hmacKey($accessId); + if ($precondition) { + self::markTestSkipped('Etag is currently not supported.'); + } + $key->update('INACTIVE', $options); + } + ], + 'storage.objects.compose' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $ob1Name = $resourceIds['objectName']; + + $bucket = self::$storageClient->bucket($bucketName); + $obj1 = $bucket->object($ob1Name); + $obj2 = $bucket->upload("line2", ["name" => "file2.txt"]); + $obj3 = $bucket->upload("test", ["name" => "combined.txt"]); + + $sourceObjects = [$ob1Name, 'file2.txt']; + if ($precondition) { + $options['ifGenerationMatch'] = $obj3->info()['generation']; + } + $bucket->compose($sourceObjects, 'combined.txt', $options); + + $obj2->delete(); + // We can't use $obj3 as it has changed, + // so we need to request the file again + $bucket->object('combined.txt')->delete(); + } + ], + 'storage.objects.copy' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + $copy = $bucket->upload("copy", ["name" => "copy.txt"]); + $options['name'] = 'copy.txt'; + if ($precondition) { + $options['ifGenerationMatch'] = $copy->info()['generation']; + } + $object->copy($bucketName, $options); + + // We can't use $copy as it has been copied over, + // so we need to request the file again + $bucket->object('copy.txt')->delete(); + } + ], + 'storage.objects.delete' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + + if ($precondition) { + $options['ifGenerationMatch'] = $object->info()['generation']; + } + + $object->delete($options); + } + ], + 'storage.objects.insert' => [ + function ($resourceIds, $options, $precondition = false, $methodGroup = null) { + if (!is_null($methodGroup)) { + self::markTestSkipped("Test only needs to run for resumable uploads"); + } + $bucketName = $resourceIds['bucketName']; + $bucket = self::$storageClient->bucket($bucketName); + + if ($precondition) { + // 0 generation for a new file + $options['ifGenerationMatch'] = 0; + } + + $options['name'] = 'file.txt'; + $object = $bucket->upload('text', $options); + + $object->delete(); + }, + function ($resourceIds, $options, $precondition = false, $methodGroup = null) { + if (!is_null($methodGroup)) { + self::markTestSkipped("Test only needs to run for resumable uploads"); + } + $bucketName = $resourceIds['bucketName']; + $bucket = self::$storageClient->bucket($bucketName); + + if ($precondition) { + // 0 generation for a new file + $options['ifGenerationMatch'] = 0; + } + + $options['name'] = 'file.txt'; + $promise = $bucket->uploadAsync('text', $options); + $object = $promise->wait(); + + $object->delete(); + }, + function ($resourceIds, $options, $precondition = false, $methodGroup = null) { + if ($methodGroup !== "storage.resumable.upload") { + self::markTestSkipped("Test only needs to run for normal uploads"); + } + $bucketName = $resourceIds['bucketName']; + $bucket = self::$storageClient->bucket($bucketName); + + $options['name'] = 'file.txt'; + $options['resumable'] = true; + $options['chunkSize'] = 512 * 1024; + if ($precondition) { + $options['ifGenerationMatch'] = 0; + } + $bucket->upload(random_bytes(16 * 1024 * 1024), $options); + + $bucket->object('file.txt')->delete(); + } + ], + 'storage.objects.patch' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + if ($precondition) { + $options['ifMetagenerationMatch'] = $object->info()['metageneration']; + } + + $object->update(['name' => 'updated.txt'], $options); + $object->delete(); + } + ], + 'storage.objects.rewrite' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + if ($precondition) { + $options['ifGenerationMatch'] = 0; + } + + $options['name'] = 'updated-file.txt'; + $object->rewrite($bucket, $options); + + $bucket->object('updated-file.txt')->delete(); + } + ], + 'storage.objects.update' => [ + // This isn't used in the library + ], + 'storage.bucket_acl.delete' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $bucket = self::$storageClient->bucket($bucketName); + + $acl = $bucket->acl(); + $list = $acl->get(); + $entity = $list[0]['entity']; + $acl->delete($entity, $options); + } + ], + 'storage.bucket_acl.insert' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $bucket = self::$storageClient->bucket($bucketName); + + $acl = $bucket->acl(); + $aclList = $acl->get(); + $entity = $aclList[0]['entity']; + $acl->add($entity, 'WRITER', $options); + } + ], + 'storage.bucket_acl.patch' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $bucket = self::$storageClient->bucket($bucketName); + + $acl = $bucket->acl(); + $aclList = $acl->get(); + $entity = $aclList[0]['entity']; + $acl->update($entity, 'READER', $options); + } + ], + 'storage.bucket_acl.update' => [ + // This isn't used in the library + ], + 'storage.default_object_acl.delete' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $bucket = self::$storageClient->bucket($bucketName); + + $acl = $bucket->defaultAcl(); + $aclList = $acl->get(); + $entity = $aclList[0]['entity']; + $acl->delete($entity, $options); + } + ], + 'storage.default_object_acl.insert' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $bucket = self::$storageClient->bucket($bucketName); + + $acl = $bucket->defaultAcl(); + $acl->add('allAuthenticatedUsers', 'OWNER', $options); + } + ], + 'storage.default_object_acl.patch' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $bucket = self::$storageClient->bucket($bucketName); + + $acl = $bucket->defaultAcl(); + $aclList = $acl->get(); + $entity = $aclList[0]['entity']; + $acl->update($entity, 'READER', $options); + } + ], + 'storage.default_object_acl.update' => [ + // This isn't used in the library + ], + 'storage.hmacKey.create' => [ + function ($resourceIds, $options, $precondition = false) { + $options['projectId'] = 'test'; + self::$storageClient->createHmacKey('temp@test.iam.gserviceaccount.com', $options); + } + ], + 'storage.notifications.insert' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $bucket = self::$storageClient->bucket($bucketName); + + $notification = $bucket->createNotification('testNotif', $options); + + $notification->delete(); + } + ], + 'storage.object_acl.delete' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + + $acl = $object->acl(); + $aclList = $acl->get(); + $entity = $aclList[0]['entity']; + $role = $aclList[0]['role']; + $acl->delete($entity, $options); + } + ], + 'storage.object_acl.insert' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + + $acl = $object->acl(); + $aclList = $acl->get(); + $entity = $aclList[0]['entity']; + $acl->add($entity, 'READER', $options); + } + ], + 'storage.object_acl.patch' => [ + function ($resourceIds, $options, $precondition = false) { + $bucketName = $resourceIds['bucketName']; + $objectName = $resourceIds['objectName']; + $bucket = self::$storageClient->bucket($bucketName); + $object = $bucket->object($objectName); + + $acl = $object->acl(); + $aclList = $acl->get(); + $entity = $aclList[0]['entity']; + $role = $aclList[0]['role']; + $acl->update($entity, 'READER', $options); + } + ], + 'storage.object_acl.update' => [ + // This isn't used in the library + ] + ]; + } + + /** + * Helper function to create the resources needed by a test. + * + * @param $resources array List of resources to create. + * @param $methodGroup string|null The group that the current + * testing operation belongs to. This can be null + * + * @return array The ids of resources created(where applicable). + */ + private function createResources(array $resources, $methodGroup) + { + $ids = []; + + // add a bucket if needed + if (isset($resources['BUCKET'])) { + $bucketName = uniqid(self::$bucketPrefix); + $bucket = self::$storageClient->createBucket($bucketName); + $ids['bucketName'] = $bucketName; + + // add the ACL roles + $acl = $bucket->acl(); + $acl->add('allUsers', 'READER'); + $acl->add('allAuthenticatedUsers', 'READER'); + + // Add the default ACL roles + $acl = $bucket->defaultAcl(); + $acl->add('allUsers', 'READER'); + $acl->add('allAuthenticatedUsers', 'READER'); + + // Create a notification for the bucket + $notifName = uniqid(self::$notificationPrefix); + $notification = $bucket->createNotification($notifName); + $ids['notificationId'] = $notification->id(); + + // Create an object if needed + if (isset($resources['OBJECT'])) { + $objectName = uniqid(self::$objectPrefix); + $content = ($methodGroup === 'storage.objects.download') ? random_bytes(10 * 1024 * 1024) : 'file text'; + $object = $bucket->upload($content, ['name' => $objectName]); + $ids['objectName'] = $objectName; + + // Create object ACL + $acl = $object->acl(); + $acl->add('allUsers', 'READER'); + $acl->add('allAuthenticatedUsers', 'READER'); + } + } + + // Create an HMAC KEY if needed. + if (isset($resources['HMAC_KEY'])) { + $keyEmail = sprintf('%s@%s.iam.gserviceaccount.com', self::$hmacKeyName, self::$projectId); + $response = self::$storageClient->createHmacKey($keyEmail); + $key = $response->hmacKey(); + $ids['hmacKeyId'] = $key->accessId(); + } + + return $ids; + } + + /** + * Helper function to dispose off the resources after a test has been performed. + * + * @param $list array The ids of the resources to destroy. + */ + private static function disposeResources(array $ids) + { + if (isset($ids['bucketName'])) { + $bucket = self::$storageClient->bucket($ids['bucketName']); + if ($bucket->exists()) { + // delete the notifications if we created any + if (isset($ids['notificationId'])) { + $notification = $bucket->notification($ids['notificationId']); + if ($notification->exists()) { + $notification->delete(); + } + } + + if (isset($ids['objectName'])) { + $object = $bucket->object($ids['objectName']); + + if ($object->exists()) { + // delete the object + $object->delete(); + } + } + + // finally delete the bucket + $bucket->delete(); + } + } + + // Dispose the hmac key if requested + if (isset($ids['hmacKeyId'])) { + $key = self::$storageClient->hmacKey($ids['hmacKeyId']); + try { + $key->update('INACTIVE'); + $key->delete(); + } catch (\Exception $e) { + // This might be thrown for a deleted key, + // for example in storage.hmacKey.delete. + // We don't have an exists method on the HmackKey class. + } + } + } +} diff --git a/Storage/tests/Conformance/data/retry_tests.json b/Storage/tests/Conformance/data/retry_tests.json new file mode 100644 index 000000000000..961ecc6fc578 --- /dev/null +++ b/Storage/tests/Conformance/data/retry_tests.json @@ -0,0 +1,275 @@ +{ + "retryTests": [ + { + "id": 1, + "description": "always_idempotent", + "cases": [ + { + "instructions": ["return-503", "return-503"] + }, + { + "instructions": ["return-reset-connection", "return-reset-connection"] + }, + { + "instructions": ["return-reset-connection", "return-503"] + } + ], + "methods": [ + {"name": "storage.bucket_acl.get", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.list", "resources": ["BUCKET"]}, + {"name": "storage.buckets.delete", "resources": ["BUCKET"]}, + {"name": "storage.buckets.get", "resources": ["BUCKET"]}, + {"name": "storage.buckets.getIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.insert", "resources": []}, + {"name": "storage.buckets.list", "resources": ["BUCKET"]}, + {"name": "storage.buckets.lockRetentionPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.testIamPermissions", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.get", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.list", "resources": ["BUCKET"]}, + {"name": "storage.hmacKey.delete", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.get", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.list", "resources": ["HMAC_KEY"]}, + {"name": "storage.notifications.delete", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.get", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.list", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.object_acl.get", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.list", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.get", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.list", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.serviceaccount.get", "resources": []} + ], + "preconditionProvided": false, + "expectSuccess": true + }, + { + "id": 2, + "description": "conditionally_idempotent_retries_when_precondition_is_present", + "cases": [ + { + "instructions": ["return-503", "return-503"] + }, + { + "instructions": ["return-reset-connection", "return-reset-connection"] + }, + { + "instructions": ["return-reset-connection", "return-503"] + } + ], + "methods": [ + {"name": "storage.buckets.patch", "resources": ["BUCKET"]}, + {"name": "storage.buckets.setIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.update", "resources": ["BUCKET"]}, + {"name": "storage.hmacKey.update", "resources": ["HMAC_KEY"]}, + {"name": "storage.objects.compose", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.copy", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.delete", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.insert", "resources": ["BUCKET"]}, + {"name": "storage.objects.patch", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.rewrite", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.update", "resources": ["BUCKET", "OBJECT"]} + ], + "preconditionProvided": true, + "expectSuccess": true + }, + { + "id": 3, + "description": "conditionally_idempotent_no_retries_when_precondition_is_absent", + "cases": [ + { + "instructions": ["return-503"] + } + ], + "methods": [ + {"name": "storage.buckets.patch", "resources": ["BUCKET"]}, + {"name": "storage.buckets.setIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.update", "resources": ["BUCKET"]}, + {"name": "storage.hmacKey.update", "resources": ["HMAC_KEY"]}, + {"name": "storage.objects.compose", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.copy", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.delete", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.insert", "resources": ["BUCKET"]}, + {"name": "storage.objects.patch", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.rewrite", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.update", "resources": ["BUCKET", "OBJECT"]} + ], + "preconditionProvided": false, + "expectSuccess": false + }, + { + "id": 4, + "description": "non_idempotent", + "cases": [ + { + "instructions": ["return-503"] + } + ], + "methods": [ + {"name": "storage.bucket_acl.delete", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.insert", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.patch", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.update", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.delete", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.insert", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.patch", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.update", "resources": ["BUCKET"]}, + {"name": "storage.hmacKey.create", "resources": []}, + {"name": "storage.notifications.insert", "resources": ["BUCKET"]}, + {"name": "storage.object_acl.delete", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.insert", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.patch", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.update", "resources": ["BUCKET", "OBJECT"]} + ], + "preconditionProvided": false, + "expectSuccess": false + }, + { + "id": 5, + "description": "non-retryable errors", + "cases": [ + { + "instructions": ["return-400"] + }, + { + "instructions": ["return-401"] + } + ], + "methods": [ + {"name": "storage.bucket_acl.delete", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.get", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.insert", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.list", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.patch", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.update", "resources": ["BUCKET"]}, + {"name": "storage.buckets.delete", "resources": ["BUCKET"]}, + {"name": "storage.buckets.get", "resources": ["BUCKET"]}, + {"name": "storage.buckets.getIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.insert", "resources": ["BUCKET"]}, + {"name": "storage.buckets.list", "resources": ["BUCKET"]}, + {"name": "storage.buckets.lockRetentionPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.patch", "resources": ["BUCKET"]}, + {"name": "storage.buckets.setIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.testIamPermissions", "resources": ["BUCKET"]}, + {"name": "storage.buckets.update", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.delete", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.get", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.insert", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.list", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.patch", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.update", "resources": ["BUCKET"]}, + {"name": "storage.hmacKey.create", "resources": []}, + {"name": "storage.hmacKey.delete", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.get", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.list", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.update", "resources": ["HMAC_KEY"]}, + {"name": "storage.notifications.delete", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.get", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.insert", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.list", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.object_acl.delete", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.get", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.insert", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.list", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.patch", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.update", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.compose", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.copy", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.delete", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.get", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.insert", "resources": ["BUCKET"]}, + {"name": "storage.objects.list", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.patch", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.rewrite", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.update", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.serviceaccount.get", "resources": []} + ], + "preconditionProvided": false, + "expectSuccess": false + }, + { + "id": 6, + "description": "mix_retryable_non_retryable_errors", + "cases": [ + { + "instructions": ["return-503", "return-400"] + }, + { + "instructions": ["return-reset-connection", "return-401"] + } + ], + "methods": [ + {"name": "storage.bucket_acl.get", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.list", "resources": ["BUCKET"]}, + {"name": "storage.buckets.delete", "resources": ["BUCKET"]}, + {"name": "storage.buckets.get", "resources": ["BUCKET"]}, + {"name": "storage.buckets.getIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.insert", "resources": []}, + {"name": "storage.buckets.list", "resources": ["BUCKET"]}, + {"name": "storage.buckets.lockRetentionPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.patch", "resources": ["BUCKET"]}, + {"name": "storage.buckets.setIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.testIamPermissions", "resources": ["BUCKET"]}, + {"name": "storage.buckets.update", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.get", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.list", "resources": ["BUCKET"]}, + {"name": "storage.hmacKey.delete", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.get", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.list", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.update", "resources": ["HMAC_KEY"]}, + {"name": "storage.notifications.delete", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.get", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.list", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.object_acl.get", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.list", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.compose", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.copy", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.delete", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.get", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.list", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.insert", "resources": ["BUCKET"]}, + {"name": "storage.objects.patch", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.rewrite", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.update", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.serviceaccount.get", "resources": []} + ], + "preconditionProvided": true, + "expectSuccess": false + }, + { + "id": 7, + "description": "resumable_uploads_handle_complex_retries", + "cases": [ + { + "instructions": ["return-reset-connection", "return-503"] + }, + { + "instructions": ["return-503-after-256K"] + }, + { + "instructions": ["return-503-after-8192K"] + } + ], + "methods": [ + {"name": "storage.objects.insert", "group": "storage.resumable.upload", "resources": ["BUCKET"]} + ], + "preconditionProvided": true, + "expectSuccess": true + }, + { + "id": 8, + "description": "downloads_handle_complex_retries", + "cases": [ + { + "instructions": ["return-broken-stream", "return-broken-stream"] + }, + { + "instructions": ["return-broken-stream-after-256K"] + } + ], + "methods": [ + {"name": "storage.objects.get", "group": "storage.objects.download", "resources": ["BUCKET", "OBJECT"]} + ], + "preconditionProvided": false, + "expectSuccess": true + } + ] + } diff --git a/Storage/tests/Snippet/StorageObjectTest.php b/Storage/tests/Snippet/StorageObjectTest.php index 631a07d0a213..3f2c41a52860 100644 --- a/Storage/tests/Snippet/StorageObjectTest.php +++ b/Storage/tests/Snippet/StorageObjectTest.php @@ -309,17 +309,17 @@ public function testDownloadAsStreamWithRangeHeaders() { $snippet = $this->snippetFromMethod(StorageObject::class, 'downloadAsStream', 1); $snippet->addLocal('object', $this->object); - $this->connection->downloadObject([ - 'restOptions' => [ - 'headers' => [ - 'Range' => 'bytes=0-4' - ] - ], - 'bucket' => 'my-bucket', - 'object' => 'my-object' - ]) - ->shouldBeCalled() - ->willReturn(Utils::streamFor('test')); + $this->connection->downloadObject(Argument::allOf( + Argument::withEntry('bucket', 'my-bucket'), + Argument::withEntry('object', 'my-object'), + Argument::withEntry('restOptions', Argument::allOf( + Argument::withEntry('headers', Argument::allOf( + Argument::withKey('Range') + )) + )) + )) + ->shouldBeCalled() + ->willReturn(Utils::streamFor('test')); $this->object->___setProperty('connection', $this->connection->reveal()); $res = $snippet->invoke(); diff --git a/Storage/tests/Unit/Connection/RestTest.php b/Storage/tests/Unit/Connection/RestTest.php index 23eade56348b..e1dfaa1b38e5 100644 --- a/Storage/tests/Unit/Connection/RestTest.php +++ b/Storage/tests/Unit/Connection/RestTest.php @@ -17,6 +17,7 @@ namespace Google\Cloud\Storage\Tests\Unit\Connection; +use Google\ApiCore\AgentHeader; use Google\Cloud\Core\RequestBuilder; use Google\Cloud\Core\RequestWrapper; use Google\Cloud\Core\Testing\TestHelpers; @@ -24,6 +25,8 @@ use Google\Cloud\Core\Upload\ResumableUploader; use Google\Cloud\Core\Upload\StreamableUploader; use Google\Cloud\Storage\Connection\Rest; +use Google\Cloud\Storage\Connection\RetryTrait; +use Google\Cloud\Storage\StorageClient; use Google\CRC32\CRC32; use GuzzleHttp\Promise; use GuzzleHttp\Promise\PromiseInterface; @@ -410,7 +413,7 @@ public function insertObjectProvider() */ public function testChooseValidationMethod($args, $extensionLoaded, $supportsBuiltin, $expected) { - $rest = new RestCrc32cStub; + $rest = new RestCrc32cStub(); $rest->extensionLoaded = $extensionLoaded; $rest->supportsBuiltin = $supportsBuiltin; @@ -469,6 +472,58 @@ public function validationMethod() ]; } + /** + * This tests whether the $arguments passed to the callbacks for header + * updation is properly done when those callbacks are invoked in the + * ExponentialBackoff::execute() method. + * + * @dataProvider provideRetryHeaders + */ + public function testRetryHeaders(int $maxAttempts) + { + $attempt = 0; + $response = new Response(200, [], $this->successBody); + $actualRequest = null; + + $httpHandler = function ($request, $options) use (&$attempt, &$actualRequest, $response, $maxAttempts) { + if (++$attempt < $maxAttempts) { + throw new \Exception('Retrying'); + } + $actualRequest = $request; + return $response; + }; + + $rest = new Rest([ + 'httpHandler' => $httpHandler, + // Mock the authHttpHandler so it doesn't make a real request + 'authHttpHandler' => function () { + return new Response(200, [], '{"access_token": "abc"}'); + }, + // Mock the delay function so the tests execute faster + 'restDelayFunction' => function () { + }, + ]); + + // Call any method to test the retry + $rest->listBuckets(); + + $this->assertNotNull($actualRequest); + $this->assertNotNull($agentHeader = $actualRequest->getHeaderLine(AgentHeader::AGENT_HEADER_KEY)); + + $agentHeaderParts = explode(' ', $agentHeader); + $this->assertStringStartsWith('gccl-invocation-id/', $agentHeaderParts[2]); + $this->assertEquals('gccl-attempt-count/' . $maxAttempts, $agentHeaderParts[3]); + } + + public function provideRetryHeaders() + { + return [ + [1], + [2], + [3], + ]; + } + private function getContentTypeAndMetadata(RequestInterface $request) { // Resumable upload request diff --git a/Storage/tests/Unit/Connection/RetryTraitTest.php b/Storage/tests/Unit/Connection/RetryTraitTest.php new file mode 100644 index 000000000000..353223bfa641 --- /dev/null +++ b/Storage/tests/Unit/Connection/RetryTraitTest.php @@ -0,0 +1,178 @@ +isPreConditionSupplied('buckets.patch', ['ifMetagenerationMatch' => 1]); + $this->assertTrue($result); + } + + /** + * Tests the falsy case of isPreconditionSupplied + * We simply pass in an operation that is conditionally idempotent + * but we don't pass any precondition or we pass an invalid + * precondition to that particular op. + */ + public function testIsPreconditionSuppliedWithInvalidPrecondition() + { + $retry = new RetryTraitImpl([]); + $result = $retry->isPreConditionSupplied('buckets.patch', []); + $this->assertFalse($result); + } + + /** + * Tests another falsy case of isPreconditionSupplied + * We simply pass in an operation that is not conditionally + * idempotent. With that it shouldn't matter if the precondition + * is actually passed or not. + */ + public function testIsPreconditionSuppliedWithInvalidOp() + { + $retry = new RetryTraitImpl([]); + $result = $retry->isPreConditionSupplied('bucket_acl.get', ['ifMetagenerationMatch' => 1]); + $this->assertFalse($result); + } + + /** + * @dataProvider retryFunctionReturnValues + */ + public function testRetryFunction( + $resource, + $op, + $restConfig, + $args, + $errorCode, + $currAttempt, + $expected + ) { + $retry = new RetryTraitImpl([]); + $retryFun = $retry->getRestRetryFunction($resource, $op, $args); + + $this->assertEquals( + $expected, + $retryFun(new \Exception('', $errorCode), $currAttempt) + ); + } + + public function retryFunctionReturnValues() + { + return [ + // Idempotent operation with retriable error code + ['buckets', 'get', [], [], 503, 1, true], + ['serviceaccount', 'get', [], [], 504, 1, true], + // Idempotent operation with non retriable error code + ['buckets', 'get', [], [], 400, 1, false], + // Conditionally Idempotent with retriable error code + // correct precondition provided + ['buckets', 'update', [], ['ifMetagenerationMatch' => 0], 503, 1, true], + // Conditionally Idempotent with retriable error code + // wrong precondition provided + ['buckets', 'update', [], ['ifGenerationMatch' => 0], 503, 1, false], + // Conditionally Idempotent with non retriable error code + // precondition provided + ['buckets', 'update', [], ['ifMetagenerationMatch' => 0], 400, 1, false], + // Conditionally Idempotent with retriable error code + // precondition not provided + ['buckets', 'update', [], [], 503, 1, false], + // Conditionally Idempotent with non retriable error code + // precondition not provided + ['buckets', 'update', [], [], 400, 1, false], + // Non idempotent + ['bucket_acl', 'delete', [], [], 503, 2, false], + ['bucket_acl', 'delete', [], [], 400, 3, false], + ]; + } + + /** + * Checks different cases for the retry strategy. + * Essentially there are 4 cases(if an error is retryable): + * - When the strategy is 'always', we retry the error, + * regardless of the operation type. + * - When the strategy is 'never' we simply don't retry ever. + * even if the op is idempotent etc. + * - When the strategy is idempotent(default), + * the decidion is based on the op context. + */ + public function retryStrategyProvider() + { + return [ + // The op is a conditionally idempotent operation, + // but it should still be retried because we pass the strategy as 'always' + [false, true, false, StorageClient::RETRY_ALWAYS, true], + // The op is an idempotent operation, + // but it should still not be retried because we pass the strategy as 'never' + [true, false, false, StorageClient::RETRY_NEVER, false], + // The op is a conditionally idempotent operation, + // so, the decision is based on the status of the precondition supplied by the user + [false, true, false, StorageClient::RETRY_IDEMPOTENT, false], + [false, true, true, StorageClient::RETRY_IDEMPOTENT, true], + ]; + } + + /** + * @dataProvider retryStrategyProvider + */ + public function testRetryStrategy( + bool $isIdempotent, + bool $condIdempotent, + bool $preconditionSupplied, + string $strategy, + bool $expected + ) { + // We intentionally pass a retryable exception + // so that the decision is completely based on the retry strategy + $retryAbleException = new \Exception("", 503); + + $retry = new RetryTraitImpl(); + $shouldRetry = $retry->retryDeciderFunction( + $retryAbleException, + $isIdempotent, + $condIdempotent, + $preconditionSupplied, + $strategy + ); + + $this->assertEquals($shouldRetry, $expected); + } +} + +//@codingStandardsIgnoreStart +class RetryTraitImpl +{ + use RetryTrait { + getRestRetryFunction as public; + isPreConditionSupplied as public; + retryDeciderFunction as public; + } +} diff --git a/Storage/tests/Unit/StorageObjectTest.php b/Storage/tests/Unit/StorageObjectTest.php index e60b1b7bf1d1..781281243613 100644 --- a/Storage/tests/Unit/StorageObjectTest.php +++ b/Storage/tests/Unit/StorageObjectTest.php @@ -17,6 +17,7 @@ namespace Google\Cloud\Storage\Tests\Unit; +use Google\ApiCore\AgentHeader; use Google\Auth\Credentials\ServiceAccountCredentials; use Google\Auth\SignBlobInterface; use Google\Cloud\Core\Exception\NotFoundException; @@ -31,7 +32,9 @@ use Google\Cloud\Storage\StorageObject; use GuzzleHttp\Promise; use GuzzleHttp\Promise\PromiseInterface; +use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Utils; +use GuzzleHttp\Exception\RequestException; use InvalidArgumentException; use PHPUnit\Framework\TestCase; use Prophecy\Argument; @@ -416,19 +419,17 @@ public function testDownloadsAsString() $bucket = 'bucket'; $object = self::OBJECT; $stream = Utils::streamFor($string = 'abcdefg'); - $this->connection->downloadObject([ - 'bucket' => $bucket, - 'object' => $object, - 'restOptions' => [ - 'headers' => [ - 'x-goog-encryption-algorithm' => 'AES256', - 'x-goog-encryption-key' => $key, - 'x-goog-encryption-key-sha256' => $hash, - ] - ] - ]) - ->willReturn($stream) - ->shouldBeCalledTimes(1); + $this->connection->downloadObject(Argument::allOf( + Argument::withEntry('bucket', $bucket), + Argument::withEntry('object', $object), + Argument::withEntry('restOptions', Argument::allOf( + Argument::withEntry('headers', Argument::allOf( + Argument::withEntry('x-goog-encryption-algorithm', 'AES256'), + Argument::withEntry('x-goog-encryption-key', $key), + Argument::withEntry('x-goog-encryption-key-sha256', $hash) + )) + )) + ))->willReturn($stream)->shouldBeCalledTimes(1); $object = new StorageObject($this->connection->reveal(), $object, $bucket); @@ -445,18 +446,17 @@ public function testDownloadsToFile() $bucket = 'bucket'; $object = self::OBJECT; $stream = Utils::streamFor($string = 'abcdefg'); - $this->connection->downloadObject([ - 'bucket' => $bucket, - 'object' => $object, - 'restOptions' => [ - 'headers' => [ - 'x-goog-encryption-algorithm' => 'AES256', - 'x-goog-encryption-key' => $key, - 'x-goog-encryption-key-sha256' => $hash, - ] - ] - ]) - ->willReturn($stream); + $this->connection->downloadObject(Argument::allOf( + Argument::withEntry('bucket', $bucket), + Argument::withEntry('object', $object), + Argument::withEntry('restOptions', Argument::allOf( + Argument::withEntry('headers', Argument::allOf( + Argument::withEntry('x-goog-encryption-algorithm', 'AES256'), + Argument::withEntry('x-goog-encryption-key', $key), + Argument::withEntry('x-goog-encryption-key-sha256', $hash) + )) + )) + ))->willReturn($stream); $object = new StorageObject($this->connection->reveal(), $object, $bucket); @@ -492,11 +492,10 @@ public function testDownloadAsStreamWithoutExtraOptions() $bucket = 'bucket'; $object = self::OBJECT; $stream = Utils::streamFor($string = 'abcdefg'); - $this->connection->downloadObject([ - 'bucket' => $bucket, - 'object' => $object, - ]) - ->willReturn($stream); + $this->connection->downloadObject(Argument::allOf( + Argument::withEntry('bucket', $bucket), + Argument::withEntry('object', $object) + ))->willReturn($stream); $object = new StorageObject($this->connection->reveal(), $object, $bucket); @@ -513,18 +512,17 @@ public function testDownloadAsStreamWithExtraOptions() $bucket = 'bucket'; $object = self::OBJECT; $stream = Utils::streamFor($string = 'abcdefg'); - $this->connection->downloadObject([ - 'bucket' => $bucket, - 'object' => $object, - 'restOptions' => [ - 'headers' => [ - 'x-goog-encryption-algorithm' => 'AES256', - 'x-goog-encryption-key' => $key, - 'x-goog-encryption-key-sha256' => $hash - ] - ] - ]) - ->willReturn($stream); + $this->connection->downloadObject(Argument::allOf( + Argument::withEntry('bucket', $bucket), + Argument::withEntry('object', $object), + Argument::withEntry('restOptions', Argument::allOf( + Argument::withEntry('headers', Argument::allOf( + Argument::withEntry('x-goog-encryption-algorithm', 'AES256'), + Argument::withEntry('x-goog-encryption-key', $key), + Argument::withEntry('x-goog-encryption-key-sha256', $hash) + )) + )) + ))->willReturn($stream); $object = new StorageObject($this->connection->reveal(), $object, $bucket); @@ -572,6 +570,71 @@ public function testDownloadAsStreamAsync() $this->assertEquals($string, $result); } + /** + * This tests whether the $arguments passed to the callbacks for header + * updation is properly done when those callbacks are invoked in the + * ExponentialBackoff::execute() method. + * + * @dataProvider provideDownloadAsStreamRetryHeaders + */ + public function testDownloadAsStreamRetryHeaders($expectedRange, $options) + { + $attempt = 0; + $responses = [ + 1 => new Response(200, [], 'ten-bytes-'), + 2 => new Response(200, [], 'twenty-bytes--------'), + ]; + $actualRequest = null; + $actualOptions = null; + + $httpHandler = function ($request, $options) use (&$attempt, &$actualRequest, &$actualOptions, $responses) { + if (++$attempt < 3) { + throw RequestException::create($request, $responses[$attempt]); + } + $actualRequest = $request; + $actualOptions = $options; + return new Response(200, [], 'ok'); + }; + + $rest = new Rest([ + 'httpHandler' => $httpHandler, + // Mock the authHttpHandler so it doesn't make a real request + 'authHttpHandler' => function () { + return new Response(200, [], '{"access_token": "abc"}'); + }, + // Mock the delay function so the tests execute faster + 'restDelayFunction' => function () { + }, + ]); + + $object = new StorageObject($rest, 'object', 'bucket'); + $stream = $object->downloadAsStream($options); + + $this->assertIsArray($actualOptions); + $this->assertArrayHasKey('headers', $actualOptions); + $this->assertArrayHasKey('Range', $actualOptions['headers']); + $this->assertEquals($expectedRange, $actualOptions['headers']['Range']); + + $this->assertNotNull($actualRequest); + $this->assertNotNull($agentHeader = $actualRequest->getHeaderLine(AgentHeader::AGENT_HEADER_KEY)); + $agentHeaderParts = explode(' ', $agentHeader); + $this->assertStringStartsWith('gccl-invocation-id/', $agentHeaderParts[2]); + $this->assertEquals('gccl-attempt-count/3', $agentHeaderParts[3]); + + // assert the resulting stream looks like we'd expect + $this->assertEquals('ten-bytes-twenty-bytes--------ok', (string) $stream); + } + + public function provideDownloadAsStreamRetryHeaders() + { + return [ + ['bytes=30-', []], + ['bytes=40-', ['restOptions' => ['headers' => ['Range' => 'bytes=10-']]]], + ['bytes=80-100', ['restOptions' => ['headers' => ['Range' => 'bytes=50-100']]]], + ['bytes=30-20', ['restOptions' => ['headers' => ['Range' => 'bytes=0-20']]]], + ]; + } + public function testGetsInfo() { $objectInfo = [