Fix systest bitrot. (#77)

Closes #67 

Note that system tests aren't being run on CircleCI, but we plan to move
to running on Kokoro.

noxfile.py

@@ -19,7 +19,6 @@
 
 
 SYSTEM_TEST_ENV_VARS = (
-    'GOOGLE_RESUMABLE_MEDIA_BUCKET',
     'GOOGLE_APPLICATION_CREDENTIALS',
 )
 REQUESTS = 'requests >= 2.18.0, < 3.0.0dev'

tests/system/requests/conftest.py

@@ -0,0 +1,61 @@
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""py.test fixtures to be shared across multiple system test modules."""
+
+import google.auth
+import google.auth.transport.requests as tr_requests
+import pytest
+
+from tests.system import utils
+
+
+def ensure_bucket(transport):
+    get_response = transport.get(utils.BUCKET_URL)
+    if get_response.status_code == 404:
+        credentials = transport.credentials
+        query_params = {
+            'project': credentials.project_id,
+        }
+        payload = {
+            'name': utils.BUCKET_NAME,
+        }
+        post_response = transport.post(
+            utils.BUCKET_POST_URL, params=query_params, json=payload)
+
+        if not post_response.ok:
+            raise ValueError("{}: {}".format(
+                post_response.status_code, post_response.reason))
+
+
+def cleanup_bucket(transport):
+    del_response = transport.delete(utils.BUCKET_URL)
+
+    if not del_response.ok:
+        raise ValueError("{}: {}".format(
+            del_response.status_code, del_response.reason))
+
+
+@pytest.fixture(scope=u'session')
+def authorized_transport():
+    credentials, _ = google.auth.default(scopes=(utils.GCS_RW_SCOPE,))
+    yield tr_requests.AuthorizedSession(credentials)
+
+
+@pytest.fixture(scope=u'session')
+def bucket(authorized_transport):
+    ensure_bucket(authorized_transport)
+
+    yield utils.BUCKET_URL
+
+    cleanup_bucket(authorized_transport)

tests/system/requests/test_download.py

@@ -78,7 +78,7 @@
     u'The content for this response was already consumed')
 NOT_FOUND_ERR = (
     b'No such object: ' +
-    bytes(os.environ['GOOGLE_RESUMABLE_MEDIA_BUCKET'], 'utf-8') +
+    utils.BUCKET_NAME.encode('utf-8') +
     b'/does-not-exist.txt'
 )
 
@@ -112,12 +112,6 @@ def request(self, method, url, data=None, headers=None, **kwargs):
         return response
 
 
-@pytest.fixture(scope=u'module')
-def authorized_transport():
-    credentials, _ = google.auth.default(scopes=(utils.GCS_RW_SCOPE,))
-    yield tr_requests.AuthorizedSession(credentials)
-
-
 # Transport that returns corrupt data, so we can exercise checksum handling.
 @pytest.fixture(scope=u'module')
 def corrupting_transport():
@@ -148,7 +142,7 @@ def _get_blob_name(info):
 
 
 @pytest.fixture(scope=u'module')
-def add_files(authorized_transport):
+def add_files(authorized_transport, bucket):
     blob_names = []
     for info in ALL_FILES:
         to_upload = _get_contents_for_upload(info)
@@ -226,6 +220,7 @@ def test_download_to_stream(add_files, authorized_transport):
         check_tombstoned(download, authorized_transport)
 
 
+@pytest.mark.xfail  # See: #76
 def test_corrupt_download(add_files, corrupting_transport):
     for info in ALL_FILES:
         blob_name = _get_blob_name(info)
@@ -246,7 +241,7 @@ def test_corrupt_download(add_files, corrupting_transport):
 
 
 @pytest.fixture(scope=u'module')
-def secret_file(authorized_transport):
+def secret_file(authorized_transport, bucket):
     blob_name = u'super-seekrit.txt'
     data = b'Please do not tell anyone my encrypted seekrit.'
 
@@ -291,7 +286,7 @@ def test_extra_headers(authorized_transport, secret_file):
     check_tombstoned(download_wo, authorized_transport)
 
 
-def test_non_existent_file(authorized_transport):
+def test_non_existent_file(authorized_transport, bucket):
     blob_name = u'does-not-exist.txt'
     media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
     download = resumable_requests.Download(media_url)
@@ -304,7 +299,7 @@ def test_non_existent_file(authorized_transport):
 
 
 @pytest.fixture(scope=u'module')
-def simple_file(authorized_transport):
+def simple_file(authorized_transport, bucket):
     blob_name = u'basic-file.txt'
     upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
     upload = resumable_requests.SimpleUpload(upload_url)
@@ -399,6 +394,7 @@ def consume_chunks(download, authorized_transport,
     return num_responses, response
 
 
+@pytest.mark.xfail  # See issue #56
 def test_chunked_download(add_files, authorized_transport):
     for info in ALL_FILES:
         actual_contents = _get_contents(info)

tests/system/requests/test_upload.py

@@ -17,8 +17,6 @@
 import io
 import os
 
-import google.auth
-import google.auth.transport.requests as tr_requests
 import pytest
 from six.moves import http_client
 from six.moves import urllib_parse
@@ -42,12 +40,6 @@
     b'1024 bytes, which does not meet this requirement.')
 
 
-@pytest.fixture(scope=u'module')
-def authorized_transport():
-    credentials, _ = google.auth.default(scopes=(utils.GCS_RW_SCOPE,))
-    yield tr_requests.AuthorizedSession(credentials)
-
-
 @pytest.fixture
 def cleanup():
     to_delete = []
@@ -116,7 +108,7 @@ def check_does_not_exist(transport, blob_name):
     assert response.status_code == http_client.NOT_FOUND
 
 
-def test_simple_upload(authorized_transport, cleanup):
+def test_simple_upload(authorized_transport, bucket, cleanup):
     with open(ICO_FILE, u'rb') as file_obj:
         actual_contents = file_obj.read()
 
@@ -139,7 +131,7 @@ def test_simple_upload(authorized_transport, cleanup):
         upload, authorized_transport, actual_contents, ICO_CONTENT_TYPE)
 
 
-def test_simple_upload_with_headers(authorized_transport, cleanup):
+def test_simple_upload_with_headers(authorized_transport, bucket, cleanup):
     blob_name = u'some-stuff.bin'
     # Make sure to clean up the uploaded blob when we are done.
     cleanup(blob_name, authorized_transport)
@@ -163,7 +155,7 @@ def test_simple_upload_with_headers(authorized_transport, cleanup):
         upload, authorized_transport, data, BYTES_CONTENT_TYPE)
 
 
-def test_multipart_upload(authorized_transport, cleanup):
+def test_multipart_upload(authorized_transport, bucket, cleanup):
     with open(ICO_FILE, u'rb') as file_obj:
         actual_contents = file_obj.read()
 
@@ -193,7 +185,7 @@ def test_multipart_upload(authorized_transport, cleanup):
         metadata, ICO_CONTENT_TYPE)
 
 
-def test_multipart_upload_with_headers(authorized_transport, cleanup):
+def test_multipart_upload_with_headers(authorized_transport, bucket, cleanup):
     blob_name = u'some-multipart-stuff.bin'
     # Make sure to clean up the uploaded blob when we are done.
     cleanup(blob_name, authorized_transport)
@@ -308,12 +300,12 @@ def _resumable_upload_helper(authorized_transport, stream, cleanup,
     check_tombstoned(upload, authorized_transport)
 
 
-def test_resumable_upload(authorized_transport, img_stream, cleanup):
+def test_resumable_upload(authorized_transport, img_stream, bucket, cleanup):
     _resumable_upload_helper(authorized_transport, img_stream, cleanup)
 
 
 def test_resumable_upload_with_headers(
-        authorized_transport, img_stream, cleanup):
+        authorized_transport, img_stream, bucket, cleanup):
     headers = utils.get_encryption_headers()
     _resumable_upload_helper(
         authorized_transport, img_stream, cleanup, headers=headers)
@@ -405,11 +397,12 @@ def _resumable_upload_recover_helper(authorized_transport, cleanup,
     check_tombstoned(upload, authorized_transport)
 
 
-def test_resumable_upload_recover(authorized_transport, cleanup):
+def test_resumable_upload_recover(authorized_transport, bucket, cleanup):
     _resumable_upload_recover_helper(authorized_transport, cleanup)
 
 
-def test_resumable_upload_recover_with_headers(authorized_transport, cleanup):
+def test_resumable_upload_recover_with_headers(
+        authorized_transport, bucket, cleanup):
     headers = utils.get_encryption_headers()
     _resumable_upload_recover_helper(
         authorized_transport, cleanup, headers=headers)
@@ -445,7 +438,8 @@ def _check_partial(self, upload, response, chunk_size, num_chunks):
         self._check_range_sent(response, start_byte, end_byte, u'*')
         self._check_range_received(response, end_byte + 1)
 
-    def test_smaller_than_chunk_size(self, authorized_transport, cleanup):
+    def test_smaller_than_chunk_size(
+            self, authorized_transport, bucket, cleanup):
         blob_name = os.path.basename(ICO_FILE)
         chunk_size = resumable_media.UPLOAD_CHUNK_SIZE
         # Make sure to clean up the uploaded blob when we are done.
@@ -480,7 +474,7 @@ def test_smaller_than_chunk_size(self, authorized_transport, cleanup):
             # Make sure the upload is tombstoned.
             check_tombstoned(upload, authorized_transport)
 
-    def test_finish_at_chunk(self, authorized_transport, cleanup):
+    def test_finish_at_chunk(self, authorized_transport, bucket, cleanup):
         blob_name = u'some-clean-stuff.bin'
         chunk_size = resumable_media.UPLOAD_CHUNK_SIZE
         # Make sure to clean up the uploaded blob when we are done.
@@ -526,7 +520,7 @@ def _add_bytes(stream, data):
         # Go back to where we were before the write.
         stream.seek(curr_pos)
 
-    def test_interleave_writes(self, authorized_transport, cleanup):
+    def test_interleave_writes(self, authorized_transport, bucket, cleanup):
         blob_name = u'some-moar-stuff.bin'
         chunk_size = resumable_media.UPLOAD_CHUNK_SIZE
         # Make sure to clean up the uploaded blob when we are done.

tests/system/utils.py

@@ -14,25 +14,24 @@
 
 import base64
 import hashlib
-import os
+import time
 
 
-BUCKET_NAME = os.environ[u'GOOGLE_RESUMABLE_MEDIA_BUCKET']
-DOWNLOAD_URL_TEMPLATE = (
-    u'https://www.googleapis.com/download/storage/v1/b/' +
-    BUCKET_NAME +
-    u'/o/{blob_name}?alt=media')
-_UPLOAD_BASE = (
-    u'https://www.googleapis.com/upload/storage/v1/b/' +
-    BUCKET_NAME +
-    u'/o?uploadType=')
+BUCKET_NAME = u'grpm-systest-{}'.format(int(1000 * time.time()))
+BUCKET_POST_URL = u'https://www.googleapis.com/storage/v1/b/'
+BUCKET_URL = u'https://www.googleapis.com/storage/v1/b/{}'.format(BUCKET_NAME)
+
+_DOWNLOAD_BASE = u'https://www.googleapis.com/download/storage/v1/b/{}'.format(
+    BUCKET_NAME)
+DOWNLOAD_URL_TEMPLATE = _DOWNLOAD_BASE + u'/o/{blob_name}?alt=media'
+
+_UPLOAD_BASE = u'https://www.googleapis.com/upload/storage/v1/b/{}'.format(
+    BUCKET_NAME) + u'/o?uploadType='
 SIMPLE_UPLOAD_TEMPLATE = _UPLOAD_BASE + u'media&name={blob_name}'
 MULTIPART_UPLOAD = _UPLOAD_BASE + u'multipart'
 RESUMABLE_UPLOAD = _UPLOAD_BASE + u'resumable'
-METADATA_URL_TEMPLATE = (
-    u'https://www.googleapis.com/storage/v1/b/' +
-    BUCKET_NAME +
-    u'/o/{blob_name}')
+
+METADATA_URL_TEMPLATE = BUCKET_URL + u'/o/{blob_name}'
 
 GCS_RW_SCOPE = u'https://www.googleapis.com/auth/devstorage.read_write'
 # Generated using random.choice() with all 256 byte choices.