diff --git a/noxfile.py b/noxfile.py index bbcaa026..36400af4 100644 --- a/noxfile.py +++ b/noxfile.py @@ -19,7 +19,6 @@ SYSTEM_TEST_ENV_VARS = ( - 'GOOGLE_RESUMABLE_MEDIA_BUCKET', 'GOOGLE_APPLICATION_CREDENTIALS', ) REQUESTS = 'requests >= 2.18.0, < 3.0.0dev' diff --git a/tests/system/requests/conftest.py b/tests/system/requests/conftest.py new file mode 100644 index 00000000..e5dd0475 --- /dev/null +++ b/tests/system/requests/conftest.py @@ -0,0 +1,61 @@ +# Copyright 2019 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""py.test fixtures to be shared across multiple system test modules.""" + +import google.auth +import google.auth.transport.requests as tr_requests +import pytest + +from tests.system import utils + + +def ensure_bucket(transport): + get_response = transport.get(utils.BUCKET_URL) + if get_response.status_code == 404: + credentials = transport.credentials + query_params = { + 'project': credentials.project_id, + } + payload = { + 'name': utils.BUCKET_NAME, + } + post_response = transport.post( + utils.BUCKET_POST_URL, params=query_params, json=payload) + + if not post_response.ok: + raise ValueError("{}: {}".format( + post_response.status_code, post_response.reason)) + + +def cleanup_bucket(transport): + del_response = transport.delete(utils.BUCKET_URL) + + if not del_response.ok: + raise ValueError("{}: {}".format( + del_response.status_code, del_response.reason)) + + +@pytest.fixture(scope=u'session') +def authorized_transport(): + credentials, _ = google.auth.default(scopes=(utils.GCS_RW_SCOPE,)) + yield tr_requests.AuthorizedSession(credentials) + + +@pytest.fixture(scope=u'session') +def bucket(authorized_transport): + ensure_bucket(authorized_transport) + + yield utils.BUCKET_URL + + cleanup_bucket(authorized_transport) diff --git a/tests/system/requests/test_download.py b/tests/system/requests/test_download.py index c4674732..ace99938 100644 --- a/tests/system/requests/test_download.py +++ b/tests/system/requests/test_download.py @@ -78,7 +78,7 @@ u'The content for this response was already consumed') NOT_FOUND_ERR = ( b'No such object: ' + - bytes(os.environ['GOOGLE_RESUMABLE_MEDIA_BUCKET'], 'utf-8') + + utils.BUCKET_NAME.encode('utf-8') + b'/does-not-exist.txt' ) @@ -112,12 +112,6 @@ def request(self, method, url, data=None, headers=None, **kwargs): return response -@pytest.fixture(scope=u'module') -def authorized_transport(): - credentials, _ = google.auth.default(scopes=(utils.GCS_RW_SCOPE,)) - yield tr_requests.AuthorizedSession(credentials) - - # Transport that returns corrupt data, so we can exercise checksum handling. @pytest.fixture(scope=u'module') def corrupting_transport(): @@ -148,7 +142,7 @@ def _get_blob_name(info): @pytest.fixture(scope=u'module') -def add_files(authorized_transport): +def add_files(authorized_transport, bucket): blob_names = [] for info in ALL_FILES: to_upload = _get_contents_for_upload(info) @@ -226,6 +220,7 @@ def test_download_to_stream(add_files, authorized_transport): check_tombstoned(download, authorized_transport) +@pytest.mark.xfail # See: #76 def test_corrupt_download(add_files, corrupting_transport): for info in ALL_FILES: blob_name = _get_blob_name(info) @@ -246,7 +241,7 @@ def test_corrupt_download(add_files, corrupting_transport): @pytest.fixture(scope=u'module') -def secret_file(authorized_transport): +def secret_file(authorized_transport, bucket): blob_name = u'super-seekrit.txt' data = b'Please do not tell anyone my encrypted seekrit.' @@ -291,7 +286,7 @@ def test_extra_headers(authorized_transport, secret_file): check_tombstoned(download_wo, authorized_transport) -def test_non_existent_file(authorized_transport): +def test_non_existent_file(authorized_transport, bucket): blob_name = u'does-not-exist.txt' media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) download = resumable_requests.Download(media_url) @@ -304,7 +299,7 @@ def test_non_existent_file(authorized_transport): @pytest.fixture(scope=u'module') -def simple_file(authorized_transport): +def simple_file(authorized_transport, bucket): blob_name = u'basic-file.txt' upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name) upload = resumable_requests.SimpleUpload(upload_url) @@ -399,6 +394,7 @@ def consume_chunks(download, authorized_transport, return num_responses, response +@pytest.mark.xfail # See issue #56 def test_chunked_download(add_files, authorized_transport): for info in ALL_FILES: actual_contents = _get_contents(info) diff --git a/tests/system/requests/test_upload.py b/tests/system/requests/test_upload.py index 07335945..61648686 100644 --- a/tests/system/requests/test_upload.py +++ b/tests/system/requests/test_upload.py @@ -17,8 +17,6 @@ import io import os -import google.auth -import google.auth.transport.requests as tr_requests import pytest from six.moves import http_client from six.moves import urllib_parse @@ -42,12 +40,6 @@ b'1024 bytes, which does not meet this requirement.') -@pytest.fixture(scope=u'module') -def authorized_transport(): - credentials, _ = google.auth.default(scopes=(utils.GCS_RW_SCOPE,)) - yield tr_requests.AuthorizedSession(credentials) - - @pytest.fixture def cleanup(): to_delete = [] @@ -116,7 +108,7 @@ def check_does_not_exist(transport, blob_name): assert response.status_code == http_client.NOT_FOUND -def test_simple_upload(authorized_transport, cleanup): +def test_simple_upload(authorized_transport, bucket, cleanup): with open(ICO_FILE, u'rb') as file_obj: actual_contents = file_obj.read() @@ -139,7 +131,7 @@ def test_simple_upload(authorized_transport, cleanup): upload, authorized_transport, actual_contents, ICO_CONTENT_TYPE) -def test_simple_upload_with_headers(authorized_transport, cleanup): +def test_simple_upload_with_headers(authorized_transport, bucket, cleanup): blob_name = u'some-stuff.bin' # Make sure to clean up the uploaded blob when we are done. cleanup(blob_name, authorized_transport) @@ -163,7 +155,7 @@ def test_simple_upload_with_headers(authorized_transport, cleanup): upload, authorized_transport, data, BYTES_CONTENT_TYPE) -def test_multipart_upload(authorized_transport, cleanup): +def test_multipart_upload(authorized_transport, bucket, cleanup): with open(ICO_FILE, u'rb') as file_obj: actual_contents = file_obj.read() @@ -193,7 +185,7 @@ def test_multipart_upload(authorized_transport, cleanup): metadata, ICO_CONTENT_TYPE) -def test_multipart_upload_with_headers(authorized_transport, cleanup): +def test_multipart_upload_with_headers(authorized_transport, bucket, cleanup): blob_name = u'some-multipart-stuff.bin' # Make sure to clean up the uploaded blob when we are done. cleanup(blob_name, authorized_transport) @@ -308,12 +300,12 @@ def _resumable_upload_helper(authorized_transport, stream, cleanup, check_tombstoned(upload, authorized_transport) -def test_resumable_upload(authorized_transport, img_stream, cleanup): +def test_resumable_upload(authorized_transport, img_stream, bucket, cleanup): _resumable_upload_helper(authorized_transport, img_stream, cleanup) def test_resumable_upload_with_headers( - authorized_transport, img_stream, cleanup): + authorized_transport, img_stream, bucket, cleanup): headers = utils.get_encryption_headers() _resumable_upload_helper( authorized_transport, img_stream, cleanup, headers=headers) @@ -405,11 +397,12 @@ def _resumable_upload_recover_helper(authorized_transport, cleanup, check_tombstoned(upload, authorized_transport) -def test_resumable_upload_recover(authorized_transport, cleanup): +def test_resumable_upload_recover(authorized_transport, bucket, cleanup): _resumable_upload_recover_helper(authorized_transport, cleanup) -def test_resumable_upload_recover_with_headers(authorized_transport, cleanup): +def test_resumable_upload_recover_with_headers( + authorized_transport, bucket, cleanup): headers = utils.get_encryption_headers() _resumable_upload_recover_helper( authorized_transport, cleanup, headers=headers) @@ -445,7 +438,8 @@ def _check_partial(self, upload, response, chunk_size, num_chunks): self._check_range_sent(response, start_byte, end_byte, u'*') self._check_range_received(response, end_byte + 1) - def test_smaller_than_chunk_size(self, authorized_transport, cleanup): + def test_smaller_than_chunk_size( + self, authorized_transport, bucket, cleanup): blob_name = os.path.basename(ICO_FILE) chunk_size = resumable_media.UPLOAD_CHUNK_SIZE # Make sure to clean up the uploaded blob when we are done. @@ -480,7 +474,7 @@ def test_smaller_than_chunk_size(self, authorized_transport, cleanup): # Make sure the upload is tombstoned. check_tombstoned(upload, authorized_transport) - def test_finish_at_chunk(self, authorized_transport, cleanup): + def test_finish_at_chunk(self, authorized_transport, bucket, cleanup): blob_name = u'some-clean-stuff.bin' chunk_size = resumable_media.UPLOAD_CHUNK_SIZE # Make sure to clean up the uploaded blob when we are done. @@ -526,7 +520,7 @@ def _add_bytes(stream, data): # Go back to where we were before the write. stream.seek(curr_pos) - def test_interleave_writes(self, authorized_transport, cleanup): + def test_interleave_writes(self, authorized_transport, bucket, cleanup): blob_name = u'some-moar-stuff.bin' chunk_size = resumable_media.UPLOAD_CHUNK_SIZE # Make sure to clean up the uploaded blob when we are done. diff --git a/tests/system/utils.py b/tests/system/utils.py index 69012157..dc4eb7a4 100644 --- a/tests/system/utils.py +++ b/tests/system/utils.py @@ -14,25 +14,24 @@ import base64 import hashlib -import os +import time -BUCKET_NAME = os.environ[u'GOOGLE_RESUMABLE_MEDIA_BUCKET'] -DOWNLOAD_URL_TEMPLATE = ( - u'https://www.googleapis.com/download/storage/v1/b/' + - BUCKET_NAME + - u'/o/{blob_name}?alt=media') -_UPLOAD_BASE = ( - u'https://www.googleapis.com/upload/storage/v1/b/' + - BUCKET_NAME + - u'/o?uploadType=') +BUCKET_NAME = u'grpm-systest-{}'.format(int(1000 * time.time())) +BUCKET_POST_URL = u'https://www.googleapis.com/storage/v1/b/' +BUCKET_URL = u'https://www.googleapis.com/storage/v1/b/{}'.format(BUCKET_NAME) + +_DOWNLOAD_BASE = u'https://www.googleapis.com/download/storage/v1/b/{}'.format( + BUCKET_NAME) +DOWNLOAD_URL_TEMPLATE = _DOWNLOAD_BASE + u'/o/{blob_name}?alt=media' + +_UPLOAD_BASE = u'https://www.googleapis.com/upload/storage/v1/b/{}'.format( + BUCKET_NAME) + u'/o?uploadType=' SIMPLE_UPLOAD_TEMPLATE = _UPLOAD_BASE + u'media&name={blob_name}' MULTIPART_UPLOAD = _UPLOAD_BASE + u'multipart' RESUMABLE_UPLOAD = _UPLOAD_BASE + u'resumable' -METADATA_URL_TEMPLATE = ( - u'https://www.googleapis.com/storage/v1/b/' + - BUCKET_NAME + - u'/o/{blob_name}') + +METADATA_URL_TEMPLATE = BUCKET_URL + u'/o/{blob_name}' GCS_RW_SCOPE = u'https://www.googleapis.com/auth/devstorage.read_write' # Generated using random.choice() with all 256 byte choices.