From ab44735ec185bedadc030ae3479cdb34251d25d4 Mon Sep 17 00:00:00 2001 From: Konstantin Kopachev Date: Thu, 16 Jan 2020 17:41:53 -0800 Subject: [PATCH] Switch from pyvows to nosetest --- Makefile | 2 +- setup.py | 7 +- tc_aws/aws/storage.py | 21 +- tests/__init__.py | 112 +++++++++ {vows => tests/fixtures}/__init__.py | 0 {vows => tests}/fixtures/image.jpg | Bin tests/fixtures/storage_fixture.py | 23 ++ tests/test_loader.py | 42 ++++ tests/test_presigning_loader.py | 78 ++++++ tests/test_result_storage.py | 154 ++++++++++++ tests/test_s3_loader.py | 97 ++++++++ tests/test_storage.py | 227 +++++++++++++++++ vows/fixtures/__init__.py | 0 vows/fixtures/storage_fixture.py | 41 ---- vows/loader_vows.py | 54 ----- vows/presigning_loader_vows.py | 102 -------- vows/result_storage_vows.py | 187 -------------- vows/s3_loader_vows.py | 121 ---------- vows/storage_vows.py | 348 --------------------------- 19 files changed, 748 insertions(+), 868 deletions(-) create mode 100644 tests/__init__.py rename {vows => tests/fixtures}/__init__.py (100%) rename {vows => tests}/fixtures/image.jpg (100%) create mode 100644 tests/fixtures/storage_fixture.py create mode 100644 tests/test_loader.py create mode 100644 tests/test_presigning_loader.py create mode 100644 tests/test_result_storage.py create mode 100644 tests/test_s3_loader.py create mode 100644 tests/test_storage.py delete mode 100644 vows/fixtures/__init__.py delete mode 100644 vows/fixtures/storage_fixture.py delete mode 100644 vows/loader_vows.py delete mode 100644 vows/presigning_loader_vows.py delete mode 100644 vows/result_storage_vows.py delete mode 100644 vows/s3_loader_vows.py delete mode 100644 vows/storage_vows.py diff --git a/Makefile b/Makefile index 1097329..db53855 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ docs: setup_docs build_docs python -mwebbrowser file:///`pwd`/docs/_build/html/index.html test: setup - pyvows -c -l tc_aws + nosetests publish: python setup.py register -r pypi diff --git a/setup.py b/setup.py index 9a88f8d..fed3d5c 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,6 @@ import re from setuptools import setup, find_packages -from setuptools.command.install import install def version(): """retrieve version from tag name""" @@ -58,16 +57,14 @@ def readme(): 'python-dateutil', 'thumbor>=6.0.0,<7', 'tornado-botocore', - #'botocore', ], extras_require={ 'tests': [ - 'pyvows', 'coverage', - 'tornado_pyvows', 'boto', - 'moto<=1.3.3', + 'moto[server]', 'mock', + 'nose', ], }, ) diff --git a/tc_aws/aws/storage.py b/tc_aws/aws/storage.py index 450aac5..74ce80c 100644 --- a/tc_aws/aws/storage.py +++ b/tc_aws/aws/storage.py @@ -87,7 +87,7 @@ def remove(self, path, callback=None): Deletes data at path :param string path: Path to delete """ - self.storage.delete(path) + self.storage.delete(path, callback) @return_future def exists(self, path, callback): @@ -109,7 +109,7 @@ def return_data(file_key): def is_expired(self, key): """ Tells whether key has expired - :param string key: Path to check + :param key: Path to check :return: Whether it is expired or not :rtype: bool """ @@ -164,13 +164,15 @@ def return_data(file_key): self.storage.get(crypto_path, callback=return_data) - def put_crypto(self, path): + @return_future + def put_crypto(self, path, callback): """ Stores crypto data at given path :param string path: Path to store the data at :return: Path where the crypto data is stored """ if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE: + callback(None) return if not self.context.server.security_key: @@ -179,9 +181,11 @@ def put_crypto(self, path): file_abspath = self._normalize_path(path) crypto_path = '%s.txt' % splitext(file_abspath)[0] - self.set(self.context.server.security_key, crypto_path) + def cb(*args, **kwargs): + callback(crypto_path) + + self.set(self.context.server.security_key, crypto_path, cb) - return crypto_path @return_future def get_detector_data(self, path, callback): @@ -202,7 +206,8 @@ def return_data(file_key): self.storage.get(path, callback=return_data) - def put_detector_data(self, path, data): + @return_future + def put_detector_data(self, path, data, callback): """ Stores detector data at given path :param string path: Path to store the data at @@ -214,9 +219,7 @@ def put_detector_data(self, path, data): path = '%s.detectors.txt' % splitext(file_abspath)[0] - self.set(dumps(data), path) - - return path + self.set(dumps(data), path, callback) def _get_error(self, response): """ diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..b186fc0 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,112 @@ +# coding: utf-8 + +# Copyright (c) 2015, thumbor-community +# Use of this source code is governed by the MIT license that can be +# found in the LICENSE file. + +import logging +import os +import signal +import subprocess as sp +import sys +import time + +import botocore.session +import mock +import requests +from botocore.client import ClientCreator +from tornado.testing import AsyncTestCase + +from tc_aws.aws.bucket import Bucket +from tests.fixtures.storage_fixture import s3_bucket + +logging.basicConfig(level=logging.CRITICAL) + +os.environ["TEST_SERVER_MODE"] = "true" + +_proxy_bypass = { + "http": None, + "https": None, +} + + +def start_service(host, port): + args = [sys.executable, "-m", "moto.server", "-H", host, + "-p", str(port)] + + process = sp.Popen(args, stderr=sp.PIPE) + url = "http://{host}:{port}".format(host=host, port=port) + + for i in range(0, 30): + if process.poll() is not None: + process.communicate() + break + + try: + # we need to bypass the proxies due to monkeypatches + requests.get(url, timeout=0.5) + break + except requests.exceptions.ConnectionError: + time.sleep(0.5) + else: + stop_process(process) + + return process + + +def stop_process(process): + try: + process.send_signal(signal.SIGTERM) + process.communicate() + except Exception: + process.kill() + outs, errors = process.communicate() + exit_code = process.returncode + msg = "Child process finished {} not in clean way: {} {}" \ + .format(exit_code, outs, errors) + raise RuntimeError(msg) + + +class FakeClientCreator(ClientCreator): + def create_client(self, *args, **kwargs): + if kwargs['endpoint_url'] is None: + kwargs['endpoint_url'] = "http://localhost:5000" + return super(FakeClientCreator, self).create_client(*args, **kwargs) + + +class S3MockedAsyncTestCase(AsyncTestCase): + _process = None + + @classmethod + def setUpClass(cls): + super(S3MockedAsyncTestCase, cls).setUpClass() + cls._process = start_service("localhost", 5000) + + os.environ['AWS_SHARED_CREDENTIALS_FILE'] = '' + os.environ['AWS_ACCESS_KEY_ID'] = 'test-key' + os.environ['AWS_SECRET_ACCESS_KEY'] = 'test-secret-key' + os.environ['AWS_SESSION_TOKEN'] = 'test-session-token' + + @classmethod + def tearDownClass(cls): + super(S3MockedAsyncTestCase, cls).tearDownClass() + stop_process(cls._process) + + def setUp(self): + super(S3MockedAsyncTestCase, self).setUp() + + requests.post("http://localhost:5000/moto-api/reset") + + self._client_patcher = mock.patch('botocore.client.ClientCreator', FakeClientCreator) + self._client_patcher.start() + + client = botocore.session.get_session().create_client('s3') + client.create_bucket(Bucket=s3_bucket) + + self.addCleanup(self._client_patcher.stop) + + def tearDown(self): + super(S3MockedAsyncTestCase, self).tearDown() + # singleton Bucket holds old IOLoop instance which closed after each test + # this cleans singleton + Bucket._instances = {} diff --git a/vows/__init__.py b/tests/fixtures/__init__.py similarity index 100% rename from vows/__init__.py rename to tests/fixtures/__init__.py diff --git a/vows/fixtures/image.jpg b/tests/fixtures/image.jpg similarity index 100% rename from vows/fixtures/image.jpg rename to tests/fixtures/image.jpg diff --git a/tests/fixtures/storage_fixture.py b/tests/fixtures/storage_fixture.py new file mode 100644 index 0000000..85054e9 --- /dev/null +++ b/tests/fixtures/storage_fixture.py @@ -0,0 +1,23 @@ +# coding: utf-8 + +# Copyright (c) 2015, thumbor-community +# Use of this source code is governed by the MIT license that can be +# found in the LICENSE file. + +from os.path import join, abspath, dirname + +from thumbor.context import ServerParameters + +s3_bucket = 'thumbor-images-test' + +IMAGE_URL = 's.glbimg.com/some/image_%s.jpg' +IMAGE_PATH = join(abspath(dirname(__file__)), 'image.jpg') + +with open(IMAGE_PATH, 'r') as img: + IMAGE_BYTES = img.read() + + +def get_server(key=None): + server_params = ServerParameters(8888, 'localhost', 'thumbor.conf', None, 'info', None) + server_params.security_key = key + return server_params diff --git a/tests/test_loader.py b/tests/test_loader.py new file mode 100644 index 0000000..ccf7875 --- /dev/null +++ b/tests/test_loader.py @@ -0,0 +1,42 @@ +# coding: utf-8 + +# Copyright (c) 2015, thumbor-community +# Use of this source code is governed by the MIT license that can be +# found in the LICENSE file. + +from unittest import TestCase + +from thumbor.config import Config +from thumbor.context import Context + +from tc_aws.loaders import _get_bucket, _get_bucket_and_key, _get_key +from .fixtures.storage_fixture import IMAGE_PATH + + +class LoaderTestCase(TestCase): + def test_can_get_bucket_and_key(self): + conf = Config( + TC_AWS_LOADER_BUCKET=None, + TC_AWS_LOADER_ROOT_PATH='' + ) + + ctx = Context(config=conf) + + path = 'some-bucket/some/image/path.jpg' + bucket, key = _get_bucket_and_key(ctx, path) + self.assertEqual(bucket, 'some-bucket') + self.assertEqual(key, 'some/image/path.jpg') + + def test_can_detect_bucket(self): + topic = _get_bucket('/'.join(['thumbor-images-test', IMAGE_PATH])) + self.assertEqual(topic, 'thumbor-images-test') + + def test_can_detect_key(self): + conf = Config( + TC_AWS_LOADER_BUCKET=None, + TC_AWS_LOADER_ROOT_PATH='', + ) + context = Context(config=conf) + key = _get_key(IMAGE_PATH, context) + + self.assertEqual(key, IMAGE_PATH) diff --git a/tests/test_presigning_loader.py b/tests/test_presigning_loader.py new file mode 100644 index 0000000..0f2a030 --- /dev/null +++ b/tests/test_presigning_loader.py @@ -0,0 +1,78 @@ +# coding: utf-8 + +# Copyright (c) 2015, thumbor-community +# Use of this source code is governed by the MIT license that can be +# found in the LICENSE file. + +from urlparse import urlparse, parse_qs + +import botocore.session +from derpconf.config import Config +from mock import patch +from thumbor.context import Context +from tornado.testing import gen_test + +from fixtures.storage_fixture import IMAGE_PATH, IMAGE_BYTES, s3_bucket +from tc_aws.loaders import presigning_loader +from tests import S3MockedAsyncTestCase + + +class PreSigningLoaderTestCase(S3MockedAsyncTestCase): + @gen_test + def test_can_load_image(self): + client = botocore.session.get_session().create_client('s3') + client.create_bucket(Bucket=s3_bucket) + + client.put_object( + Bucket=s3_bucket, + Key=''.join(['root_path', IMAGE_PATH]), + Body=IMAGE_BYTES, + ContentType='image/jpeg', ) + + conf = Config( + TC_AWS_LOADER_BUCKET=s3_bucket, + TC_AWS_LOADER_ROOT_PATH='root_path', + ) + + image = yield presigning_loader.load(Context(config=conf), IMAGE_PATH) + self.assertEqual(image.buffer, IMAGE_BYTES) + + @patch('thumbor.loaders.http_loader.load_sync') + @gen_test + def test_should_use_http_loader(self, load_sync_patch): + def cb(a, b, callback, *args, **kwargs): + callback('foobar') + return None + + load_sync_patch.side_effect = cb + + conf = Config(TC_AWS_ENABLE_HTTP_LOADER=True) + presigning_loader.load(Context(config=conf), 'http://foo.bar') + self.assertTrue(load_sync_patch.called) + + @gen_test + def test_can_validate_buckets(self): + conf = Config( + TC_AWS_ALLOWED_BUCKETS=['whitelist_bucket'], + TC_AWS_LOADER_BUCKET=None, + ) + + image = yield presigning_loader.load(Context(config=conf), '/'.join([s3_bucket, IMAGE_PATH])) + self.assertIsNone(image) + + @gen_test + def test_can_build_presigned_url(self): + context = Context(config=(Config())) + url = yield presigning_loader._generate_presigned_url(context, "bucket-name", "some-s3-key") + + url = urlparse(url) + self.assertEqual(url.scheme[0:4], 'http') + self.assertEqual(url.path, '/bucket-name/some-s3-key') + + url_params = parse_qs(url.query) + # We can't test Expires & Signature values as they vary depending on the TZ + self.assertIn('Expires', url_params) + self.assertIn('Signature', url_params) + + self.assertDictContainsSubset({'AWSAccessKeyId': ['test-key'], 'x-amz-security-token': ['test-session-token']}, + url_params) diff --git a/tests/test_result_storage.py b/tests/test_result_storage.py new file mode 100644 index 0000000..79fc585 --- /dev/null +++ b/tests/test_result_storage.py @@ -0,0 +1,154 @@ +# coding: utf-8 + +# Copyright (c) 2015, thumbor-community +# Use of this source code is governed by the MIT license that can be +# found in the LICENSE file. + +from datetime import datetime, timedelta +from hashlib import sha1 +from unittest import TestCase + +import botocore.session +from dateutil.tz import tzutc +from thumbor.config import Config +from thumbor.context import Context +from tornado.testing import gen_test + +from fixtures.storage_fixture import IMAGE_BYTES, get_server, s3_bucket +from tc_aws.result_storages.s3_storage import Storage +from tests import S3MockedAsyncTestCase + + +class Request(object): + url = None + + +class S3StorageTestCase(S3MockedAsyncTestCase): + + @gen_test + def test_can_store_image(self): + config = Config(TC_AWS_RESULT_STORAGE_BUCKET=s3_bucket) + ctx = Context(config=config, server=get_server('ACME-SEC')) + ctx.request = Request + ctx.request.url = 'foo/my-image.jpg' + + storage = Storage(ctx) + + yield storage.put(IMAGE_BYTES) + + client = botocore.session.get_session().create_client('s3') + response = client.get_object(Bucket=s3_bucket, Key="foo/my-image.jpg") + + self.assertEqual(response['Body'].read(), IMAGE_BYTES) + + @gen_test + def test_can_store_image_randomly(self): + config = Config(TC_AWS_RESULT_STORAGE_BUCKET=s3_bucket, TC_AWS_RANDOMIZE_KEYS=True) + ctx = Context(config=config, server=get_server('ACME-SEC')) + ctx.request = Request + ctx.request.url = 'foo/my-image.jpg' + + storage = Storage(ctx) + + yield storage.put(IMAGE_BYTES) + + expected_path = '/'.join([sha1("foo/my-image.jpg".encode('utf-8')).hexdigest(), 'foo/my-image.jpg']) + + client = botocore.session.get_session().create_client('s3') + response = client.get_object(Bucket=s3_bucket, Key=expected_path) + + self.assertEqual(response['Body'].read(), IMAGE_BYTES) + + @gen_test + def test_can_get_image(self): + config = Config(TC_AWS_RESULT_STORAGE_BUCKET=s3_bucket) + ctx = Context(config=config, server=get_server('ACME-SEC')) + ctx.request = Request + ctx.request.url = 'my-image-2.jpg' + + storage = Storage(ctx) + yield storage.put(IMAGE_BYTES) + + topic = yield storage.get() + + self.assertEqual(topic.buffer, IMAGE_BYTES) + + @gen_test + def test_can_get_randomized_image(self): + config = Config(TC_AWS_RESULT_STORAGE_BUCKET=s3_bucket, TC_AWS_RANDOMIZE_KEYS=True) + ctx = Context(config=config, server=get_server('ACME-SEC')) + ctx.request = Request + ctx.request.url = 'my-image-2.jpg' + + storage = Storage(ctx) + yield storage.put(IMAGE_BYTES) + + topic = yield storage.get() + + self.assertEqual(topic.buffer, IMAGE_BYTES) + + @gen_test + def test_can_get_image_with_metadata(self): + config = Config(TC_AWS_RESULT_STORAGE_BUCKET=s3_bucket, TC_AWS_STORE_METADATA=True) + ctx = Context(config=config, server=get_server('ACME-SEC')) + ctx.headers = {'Content-Type': 'image/webp', 'Some-Other-Header': 'doge-header'} + ctx.request = Request + ctx.request.url = 'my-image-meta.jpg' + + storage = Storage(ctx) + yield storage.put(IMAGE_BYTES) + + file_abspath = storage._normalize_path(ctx.request.url) + topic = yield storage.get(file_abspath) + + self.assertIn('Some-Other-Header', topic.metadata['Metadata']) + self.assertEqual(topic.metadata['Metadata']['Content-Type'], 'image/webp') + self.assertEqual(topic.buffer, IMAGE_BYTES) + + +class ExpiredTestCase(TestCase): + + @property + def expired_enabled(self): + return Storage(Context(config=Config(RESULT_STORAGE_EXPIRATION_SECONDS=3600))) + + def test_should_check_invalid_key(self): + self.assertTrue(self.expired_enabled.is_expired(None)) + self.assertTrue(self.expired_enabled.is_expired(False)) + self.assertTrue(self.expired_enabled.is_expired(dict())) + self.assertTrue(self.expired_enabled.is_expired({'Error': ''})) + + def test_should_tell_when_not_expired(self): + key = { + 'LastModified': datetime.now(tzutc()), + 'Body': 'foobar', + } + self.assertFalse(self.expired_enabled.is_expired(key)) + + def test_should_tell_when_expired(self): + key = { + 'LastModified': (datetime.now(tzutc()) - timedelta(seconds=3601)), + 'Body': 'foobar', + } + self.assertTrue(self.expired_enabled.is_expired(key)) + + def test_expire_disabled_should_not_tell_when_expired(self): + topic = Storage(Context(config=Config(RESULT_STORAGE_EXPIRATION_SECONDS=0))) + key = { + 'LastModified': (datetime.now(tzutc()) - timedelta(seconds=3601)), + 'Body': 'foobar', + } + self.assertFalse(topic.is_expired(key)) + + +class PrefixTestCase(TestCase): + + def test_should_check_invalid_key(self): + config = Config(TC_AWS_RESULT_STORAGE_BUCKET=s3_bucket, TC_AWS_RESULT_STORAGE_ROOT_PATH='tata') + ctx = Context(config=config, server=get_server('ACME-SEC')) + + storage = Storage(ctx) + + topic = storage._normalize_path('toto') + + self.assertEqual(topic, "tata/toto") diff --git a/tests/test_s3_loader.py b/tests/test_s3_loader.py new file mode 100644 index 0000000..4da14c6 --- /dev/null +++ b/tests/test_s3_loader.py @@ -0,0 +1,97 @@ +# coding: utf-8 + +# Copyright (c) 2015, thumbor-community +# Use of this source code is governed by the MIT license that can be +# found in the LICENSE file. + +import botocore.session +from derpconf.config import Config +from mock import patch, MagicMock +from thumbor.context import Context +from tornado.testing import gen_test + +from fixtures.storage_fixture import IMAGE_PATH, IMAGE_BYTES, s3_bucket +from tc_aws.loaders import s3_loader +from tests import S3MockedAsyncTestCase + + +class S3LoaderTestCase(S3MockedAsyncTestCase): + + @gen_test + def test_can_load_image(self): + client = botocore.session.get_session().create_client('s3') + client.create_bucket(Bucket=s3_bucket) + + client.put_object( + Bucket=s3_bucket, + Key=''.join(['root_path', IMAGE_PATH]), + Body=IMAGE_BYTES, + ContentType='image/jpeg', ) + + conf = Config( + TC_AWS_LOADER_BUCKET=s3_bucket, + TC_AWS_LOADER_ROOT_PATH='root_path' + ) + + image = yield s3_loader.load(Context(config=conf), IMAGE_PATH) + self.assertEqual(image, IMAGE_BYTES) + + @gen_test + def test_can_validate_buckets(self): + conf = Config( + TC_AWS_ALLOWED_BUCKETS=['whitelist_bucket'], + TC_AWS_LOADER_BUCKET=None, + ) + + image = yield s3_loader.load(Context(config=conf), '/'.join([s3_bucket, IMAGE_PATH])) + self.assertIsNone(image.buffer) + + @patch('thumbor.loaders.http_loader.load_sync') + @gen_test + def test_should_use_http_loader(self, load_sync_patch): + def cb(a, b, callback, *args, **kwargs): + callback('foobar') + return None + + load_sync_patch.side_effect = cb + + conf = Config(TC_AWS_ENABLE_HTTP_LOADER=True) + s3_loader.load(Context(config=conf), 'http://foo.bar') + self.assertTrue(load_sync_patch.called) + + @patch('thumbor.loaders.http_loader.load_sync') + @gen_test + def test_should_not_use_http_loader_if_not_prefixed_with_scheme(self, load_sync_patch): + conf = Config(TC_AWS_ENABLE_HTTP_LOADER=True) + yield s3_loader.load(Context(config=conf), 'foo/bar') + self.assertFalse(load_sync_patch.called) + + def test_datafunc_loader(self): + def callback(*args, **kwargs): + pass + + file_key = { + 'Error': 'Error', + 'ResponseMetadata': { + 'HTTPStatusCode': 502 + } + } + + self.call_count = 0 + + def get(key, callback=None): + self.call_count += 1 + callback(file_key) + + mock_bucket_loader = MagicMock() + mock_bucket_loader.get = get + + func = s3_loader.HandleDataFunc.as_func( + '/'.join([s3_bucket, IMAGE_PATH]), + callback=callback, + bucket_loader=mock_bucket_loader, + max_retry=3 + ) + + func(file_key) + self.assertEqual(self.call_count, 3) diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..686294f --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,227 @@ +# se!/usr/bin/python +# -*- coding: utf-8 -*- +# coding: utf-8 + +# Copyright (c) 2015, thumbor-community +# Use of this source code is governed by the MIT license that can be +# found in the LICENSE file. + +from datetime import datetime, timedelta +from unittest import TestCase + +from dateutil.tz import tzutc +from nose.tools import assert_raises_regexp +from thumbor.config import Config +from thumbor.context import Context, RequestParameters +from tornado.testing import gen_test + +from fixtures.storage_fixture import IMAGE_URL, IMAGE_BYTES, get_server, s3_bucket +from tc_aws.storages.s3_storage import Storage +from tests import S3MockedAsyncTestCase + + +class S3StorageTestCase(S3MockedAsyncTestCase): + + @gen_test + def test_can_store_image(self): + config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + + yield storage.put(IMAGE_URL % '1', IMAGE_BYTES) + + topic = yield storage.get(IMAGE_URL % '1') + + self.assertEqual(topic, IMAGE_BYTES) + + @gen_test + def test_can_get_image_existance(self): + config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + + yield storage.put(IMAGE_URL % '3', IMAGE_BYTES) + topic = yield storage.exists(IMAGE_URL % '3') + + self.assertTrue(topic) + + @gen_test + def test_can_get_image_inexistance(self): + config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + + topic = yield storage.exists(IMAGE_URL % '9999') + + self.assertFalse(topic) + + @gen_test + def test_can_remove_instance(self): + config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + yield storage.put(IMAGE_URL % '4', IMAGE_BYTES) + yield storage.remove(IMAGE_URL % '4') + topic = yield storage.exists(IMAGE_URL % '4') + + self.assertFalse(topic) + + @gen_test + def test_can_remove_then_put_image(self): + config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + yield storage.put(IMAGE_URL % '5', IMAGE_BYTES) + + created = yield storage.exists(IMAGE_URL % '5') + self.assertTrue(created) + + yield storage.remove(IMAGE_URL % '5') + exists = yield storage.exists(IMAGE_URL % '5') + self.assertFalse(exists) + + yield storage.put(IMAGE_URL % '5', IMAGE_BYTES) + exists = yield storage.exists(IMAGE_URL % '5') + self.assertTrue(exists) + + def test_can_return_path(self): + config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + topic = storage.resolve_original_photo_path("toto") + self.assertEqual(topic, 'toto') + + def test_should_return_storage_prefix(self): + config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket, TC_AWS_STORAGE_ROOT_PATH='tata') + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + topic = storage._normalize_path('toto') + self.assertEqual(topic, "tata/toto") + + def should_normalize_slash(self): + config = Config(TC_AWS_STORAGE_ROOT_PATH='', TC_AWS_ROOT_IMAGE_NAME='root_image') + storage = Storage(Context(config=config)) + self.assertEqual(storage._normalize_path('/test'), 'test') + self.assertEqual(storage._normalize_path('/test/'), 'test/root_image') + self.assertEqual(storage._normalize_path('/test/image.png'), 'test/image.png') + + +class CryptoS3StorageTestCase(S3MockedAsyncTestCase): + + @gen_test + def test_should_raise_on_invalid_config(self): + config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket, STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True) + storage = Storage(Context(config=config, server=get_server(''))) + + yield storage.put(IMAGE_URL % '9999', IMAGE_BYTES) + + with assert_raises_regexp(RuntimeError, + "STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified"): + storage.put_crypto(IMAGE_URL % '9999') + + @gen_test + def test_getting_crypto_for_a_new_image_returns_none(self): + config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket, STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + topic = yield storage.get_crypto(IMAGE_URL % '9999') + self.assertIsNone(topic) + + @gen_test + def test_does_not_store_if_config_says_not_to(self): + config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket, STORES_CRYPTO_KEY_FOR_EACH_IMAGE=False) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + yield storage.put(IMAGE_URL % '9998', IMAGE_BYTES) + yield storage.put_crypto(IMAGE_URL % '9998') + topic = yield storage.get_crypto(IMAGE_URL % '9998') + self.assertIsNone(topic) + + @gen_test + def test_can_store_crypto(self): + config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket, STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + yield storage.put(IMAGE_URL % '6', IMAGE_BYTES) + yield storage.put_crypto(IMAGE_URL % '6') + topic = yield storage.get_crypto(IMAGE_URL % '6') + + self.assertIsNotNone(topic) + self.assertNotIsInstance(topic, BaseException) + self.assertEqual(topic.read(), 'ACME-SEC') + + +class DetectorS3StorageTestCase(S3MockedAsyncTestCase): + + @gen_test + def test_can_store_detector_data(self): + config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + yield storage.put(IMAGE_URL % '7', IMAGE_BYTES) + yield storage.put_detector_data(IMAGE_URL % '7', 'some-data') + topic = yield storage.get_detector_data(IMAGE_URL % '7') + + self.assertEqual(topic, 'some-data') + + @gen_test + def test_returns_none_if_no_detector_data(self): + config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + topic = yield storage.get_detector_data(IMAGE_URL % '9999') + + self.assertIsNone(topic) + + +class WebpS3StorageTestCase(TestCase): + + def test_has_config_request(self): + config = Config(AUTO_WEBP=True) + context = Context(config=config) + context.request = RequestParameters(accepts_webp=True) + storage = Storage(context) + self.assertTrue(storage.is_auto_webp) + + def test_has_config_no_request(self): + config = Config(AUTO_WEBP=True) + context = Context(config=config) + storage = Storage(context) + self.assertFalse(storage.is_auto_webp) + + def test_has_config_request_does_not_accept(self): + config = Config(AUTO_WEBP=True) + context = Context(config=config) + context.request = RequestParameters(accepts_webp=False) + storage = Storage(context) + self.assertFalse(storage.is_auto_webp) + + def test_has_no_config(self): + config = Config(AUTO_WEBP=False) + context = Context(config=config) + context.request = RequestParameters(accepts_webp=True) + storage = Storage(context) + self.assertFalse(storage.is_auto_webp) + + +class ExpiredTestCase(TestCase): + + @property + def expired_enabled(self): + return Storage(Context(config=Config(STORAGE_EXPIRATION_SECONDS=3600))) + + def test_should_check_invalid_key(self): + self.assertTrue(self.expired_enabled.is_expired(None)) + self.assertTrue(self.expired_enabled.is_expired(False)) + self.assertTrue(self.expired_enabled.is_expired(dict())) + self.assertTrue(self.expired_enabled.is_expired({'Error': ''})) + + def test_should_tell_when_not_expired(self): + key = { + 'LastModified': datetime.now(tzutc()), + 'Body': 'foobar', + } + self.assertFalse(self.expired_enabled.is_expired(key)) + + def test_should_tell_when_expired(self): + key = { + 'LastModified': (datetime.now(tzutc()) - timedelta(seconds=3601)), + 'Body': 'foobar', + } + self.assertTrue(self.expired_enabled.is_expired(key)) + + def test_expire_disabled_should_not_tell_when_expired(self): + topic = Storage(Context(config=Config(STORAGE_EXPIRATION_SECONDS=0))) + key = { + 'LastModified': (datetime.now(tzutc()) - timedelta(seconds=3601)), + 'Body': 'foobar', + } + self.assertFalse(topic.is_expired(key)) diff --git a/vows/fixtures/__init__.py b/vows/fixtures/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/vows/fixtures/storage_fixture.py b/vows/fixtures/storage_fixture.py deleted file mode 100644 index ac82402..0000000 --- a/vows/fixtures/storage_fixture.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# thumbor imaging service -# https://github.com/globocom/thumbor/wiki - -# Licensed under the MIT license: -# http://www.opensource.org/licenses/mit-license -# Copyright (c) 2011 globo.com timehome@corp.globo.com - -from os.path import join, abspath, dirname - -from thumbor.context import ServerParameters, Context -from thumbor.config import Config -from thumbor.importer import Importer - -IMAGE_URL = 's.glbimg.com/some/image_%s.jpg' -IMAGE_PATH = join(abspath(dirname(__file__)), 'image.jpg') - -with open(IMAGE_PATH, 'r') as img: - IMAGE_BYTES = img.read() - - -def get_server(key=None): - server_params = ServerParameters(8888, 'localhost', 'thumbor.conf', None, 'info', None) - server_params.security_key = key - return server_params - - -def get_context(server=None, config=None, importer=None): - if not server: - server = get_server() - - if not config: - config = Config() - - if not importer: - importer = Importer(config) - - ctx = Context(server=server, config=config, importer=importer) - return ctx diff --git a/vows/loader_vows.py b/vows/loader_vows.py deleted file mode 100644 index 35c77c8..0000000 --- a/vows/loader_vows.py +++ /dev/null @@ -1,54 +0,0 @@ -# se!/usr/bin/python -# -*- coding: utf-8 -*- -from mock import Mock - -from pyvows import Vows, expect - -from thumbor.context import Context -from derpconf.config import Config - -from fixtures.storage_fixture import IMAGE_PATH - -from tc_aws.loaders import * - -import logging -logging.getLogger('botocore').setLevel(logging.CRITICAL) - -s3_bucket = 'thumbor-images-test' - -@Vows.batch -class S3LoaderVows(Vows.Context): - - class CanGetBucketAndKey(Vows.Context): - - def topic(self): - conf = Config() - conf.TC_AWS_LOADER_BUCKET = None - conf.TC_AWS_LOADER_ROOT_PATH = '' - return Context(config=conf) - - def should_detect_bucket_and_key(self, topic): - path = 'some-bucket/some/image/path.jpg' - bucket, key = _get_bucket_and_key(topic, path) - expect(bucket).to_equal('some-bucket') - expect(key).to_equal('some/image/path.jpg') - - class CanDetectBucket(Vows.Context): - - def topic(self): - return _get_bucket('/'.join([s3_bucket, IMAGE_PATH])) - - def should_detect_bucket(self, topic): - expect(topic).to_equal(s3_bucket) - - class CanDetectKey(Vows.Context): - - def topic(self): - conf = Config() - conf.TC_AWS_LOADER_BUCKET = None - conf.TC_AWS_LOADER_ROOT_PATH = '' - context = Context(config=conf) - return _get_key(IMAGE_PATH, context) - - def should_detect_key(self, topic): - expect(topic).to_equal(IMAGE_PATH) diff --git a/vows/presigning_loader_vows.py b/vows/presigning_loader_vows.py deleted file mode 100644 index 3af236e..0000000 --- a/vows/presigning_loader_vows.py +++ /dev/null @@ -1,102 +0,0 @@ -# se!/usr/bin/python -# -*- coding: utf-8 -*- - -from urlparse import urlparse, parse_qs - -import os - -from pyvows import Vows, expect -from mock import patch - -from thumbor.context import Context -from derpconf.config import Config - -import boto -from boto.s3.key import Key - -from moto import mock_s3_deprecated as mock_s3 - -from fixtures.storage_fixture import IMAGE_PATH, IMAGE_BYTES - -from tc_aws.loaders import presigning_loader - -import logging -logging.getLogger('botocore').setLevel(logging.CRITICAL) - -s3_bucket = 'thumbor-images-test' - -@Vows.batch -class PresigningLoaderVows(Vows.Context): - - class CanLoadImage(Vows.Context): - - @mock_s3 - def topic(self): - conn = boto.connect_s3() - bucket = conn.create_bucket(s3_bucket) - - k = Key(bucket) - k.key = IMAGE_PATH - k.set_contents_from_string(IMAGE_BYTES) - - conf = Config() - conf.define('TC_AWS_LOADER_BUCKET', s3_bucket, '') - conf.define('TC_AWS_LOADER_ROOT_PATH', 'root_path', '') - - return Context(config=conf) - - def should_load_from_s3(self, topic): - image = yield presigning_loader.load(topic, '/'.join(['root_path', IMAGE_PATH])) - expect(image).to_equal(IMAGE_BYTES) - - class ValidatesBuckets(Vows.Context): - - @mock_s3 - def topic(self): - conf = Config() - conf.define('TC_AWS_ALLOWED_BUCKETS', [], '') - - return Context(config=conf) - - def should_load_from_s3(self, topic): - image = yield presigning_loader.load(topic, '/'.join([s3_bucket, IMAGE_PATH])) - expect(image).to_equal(None) - - class HandlesHttpLoader(Vows.Context): - - @mock_s3 - def topic(self): - conf = Config() - conf.define('TC_AWS_ENABLE_HTTP_LOADER', True, '') - - return Context(config=conf) - - @patch('thumbor.loaders.http_loader.load_sync') - def should_redirect_to_http(self, topic, load_sync_patch): - def callback(*args): - pass - - presigning_loader.load(topic, 'http://foo.bar', callback) - expect(load_sync_patch.called).to_be_true() - - class CanBuildPresignedUrl(Vows.Context): - - @Vows.async_topic - @mock_s3 - def topic(self, callback): - os.environ['AWS_SHARED_CREDENTIALS_FILE'] = '' - conf = Config() - context = Context(config=conf) - presigning_loader._generate_presigned_url(context, "bucket-name", "some-s3-key", callback) - - def should_generate_presigned_urls(self, topic): - url = urlparse(topic.args[0]) - expect(url.scheme).to_equal('https') - expect(url.hostname).to_equal('bucket-name.s3.amazonaws.com') - expect(url.path).to_equal('/some-s3-key') - url_params = parse_qs(url.query) - # We can't test Expires & Signature values as they vary depending on the TZ - expect(url_params).to_include('Expires') - expect(url_params).to_include('Signature') - expect(url_params['AWSAccessKeyId'][0]).to_equal('test-key') - expect(url_params['x-amz-security-token'][0]).to_equal('test-session-token') diff --git a/vows/result_storage_vows.py b/vows/result_storage_vows.py deleted file mode 100644 index 4da62e4..0000000 --- a/vows/result_storage_vows.py +++ /dev/null @@ -1,187 +0,0 @@ -#se!/usr/bin/python -# -*- coding: utf-8 -*- -from datetime import datetime, timedelta -from dateutil.tz import tzutc - -from hashlib import sha1 - -from pyvows import Vows, expect - -from thumbor.context import Context -from tc_aws import Config -from fixtures.storage_fixture import IMAGE_BYTES, get_server - -from boto.s3.connection import S3Connection -from moto import mock_s3_deprecated as mock_s3 - -from tc_aws.result_storages.s3_storage import Storage - -import logging -logging.getLogger('botocore').setLevel(logging.CRITICAL) - -s3_bucket = 'thumbor-images-test' - - -class Request(object): - url = None - - -@Vows.batch -class S3ResultStorageVows(Vows.Context): - - class CanStoreImage(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_RESULT_STORAGE_BUCKET=s3_bucket) - ctx = Context(config=config, server=get_server('ACME-SEC')) - ctx.request = Request - ctx.request.url = 'my-image.jpg' - - storage = Storage(ctx) - - storage.put(IMAGE_BYTES, callback=callback) - - def should_be_in_catalog(self, topic): - expect(topic.args[0]).to_equal('my-image.jpg') - - class CanStoreImageRandomly(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_RESULT_STORAGE_BUCKET=s3_bucket, TC_AWS_RANDOMIZE_KEYS=True) - ctx = Context(config=config, server=get_server('ACME-SEC')) - ctx.request = Request - ctx.request.url = 'my-image.jpg' - - storage = Storage(ctx) - - storage.put(IMAGE_BYTES, callback=callback) - - def should_be_in_catalog(self, topic): - expect(topic.args[0]).to_equal([sha1('my-image.jpg'.encode('utf8')).hexdigest(), 'my-image.jpg'].join('/')) - - class CanGetImage(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_RESULT_STORAGE_BUCKET=s3_bucket) - ctx = Context(config=config, server=get_server('ACME-SEC')) - ctx.request = Request - ctx.request.url = 'my-image-2.jpg' - - storage = Storage(ctx) - storage.put(IMAGE_BYTES) - - storage.get(callback=callback) - - def should_have_proper_bytes(self, topic): - expect(topic.args[0]).not_to_be_null() - expect(topic.args[0]).not_to_be_an_error() - expect(topic.args[0]).to_equal(IMAGE_BYTES) - - class CanGetRandomizedImage(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_RESULT_STORAGE_BUCKET=s3_bucket, TC_AWS_RANDOMIZE_KEYS=True) - ctx = Context(config=config, server=get_server('ACME-SEC')) - ctx.request = Request - ctx.request.url = 'my-image-2.jpg' - - storage = Storage(ctx) - storage.put(IMAGE_BYTES) - - storage.get(callback=callback) - - def should_have_proper_bytes(self, topic): - expect(topic.args[0]).not_to_be_null() - expect(topic.args[0]).not_to_be_an_error() - expect(topic.args[0]).to_equal(IMAGE_BYTES) - - class CanGetImageWithMetadata(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_RESULT_STORAGE_BUCKET=s3_bucket, TC_AWS_STORE_METADATA=True) - ctx = Context(config=config, server=get_server('ACME-SEC')) - ctx.headers = {'Content-Type': 'image/webp', 'Some-Other-Header': 'doge-header'} - ctx.request = Request - ctx.request.url = 'my-image-meta.jpg' - - storage = Storage(ctx) - storage.put(IMAGE_BYTES) - - file_abspath = storage._normalize_path(ctx.request.url) - storage.storage.get(file_abspath, callback=callback) - - def should_have_proper_bytes(self, topic): - expect(topic.args[0].content_type).to_include('image/webp') - expect(topic.args[0].metadata).to_include('some-other-header') - expect(topic.args[0].content_type).to_equal(IMAGE_BYTES) - - class ExpiredVows(Vows.Context): - class WhenExpiredEnabled(Vows.Context): - def topic(self): - return Storage(Context(config=Config(RESULT_STORAGE_EXPIRATION_SECONDS=3600))) - - def should_check_invalid_key(self, topic): - expect(topic.is_expired(None)).to_be_true() - expect(topic.is_expired(False)).to_be_true() - expect(topic.is_expired(dict())).to_be_true() - expect(topic.is_expired({'Error': ''})).to_be_true() - - def should_tell_when_not_expired(self, topic): - key = { - 'LastModified': datetime.now(tzutc()), - 'Body': 'foobar', - } - expect(topic.is_expired(key)).to_be_false() - - def should_tell_when_expired(self, topic): - key = { - 'LastModified': (datetime.now(tzutc()) - timedelta(seconds=3601)), - 'Body': 'foobar', - } - expect(topic.is_expired(key)).to_be_true() - - class WhenExpiredDisabled(Vows.Context): - def topic(self): - return Storage(Context(config=Config(RESULT_STORAGE_EXPIRATION_SECONDS=0))) - - def should_not_tell_when_expired(self, topic): - key = { - 'LastModified': (datetime.now(tzutc()) - timedelta(seconds=3601)), - 'Body': 'foobar', - } - expect(topic.is_expired(key)).to_be_false() - class HandlesStoragePrefix(Vows.Context): - @mock_s3 - def topic(self): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_RESULT_STORAGE_BUCKET=s3_bucket, TC_AWS_RESULT_STORAGE_ROOT_PATH='tata') - ctx = Context(config=config, server=get_server('ACME-SEC')) - - storage = Storage(ctx) - - return storage._normalize_path('toto') - - def should_return_the_same(self, topic): - expect(topic).to_equal("tata/toto") diff --git a/vows/s3_loader_vows.py b/vows/s3_loader_vows.py deleted file mode 100644 index 1d1b34f..0000000 --- a/vows/s3_loader_vows.py +++ /dev/null @@ -1,121 +0,0 @@ -# -*- coding: utf-8 -*- -from mock import Mock - -from pyvows import Vows, expect -from mock import patch, MagicMock - -from thumbor.context import Context -from derpconf.config import Config - -import boto -from boto.s3.key import Key - -from moto import mock_s3_deprecated as mock_s3 - -from fixtures.storage_fixture import IMAGE_PATH, IMAGE_BYTES - -from tc_aws.loaders import s3_loader - -import logging -logging.getLogger('botocore').setLevel(logging.CRITICAL) - -s3_bucket = 'thumbor-images-test' - -@Vows.batch -class S3LoaderVows(Vows.Context): - - class CanLoadImage(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - conn = boto.connect_s3() - bucket = conn.create_bucket(s3_bucket) - - k = Key(bucket) - k.key = '/'.join(['root_path', IMAGE_PATH]) - k.set_contents_from_string(IMAGE_BYTES) - - conf = Config() - conf.define('TC_AWS_LOADER_BUCKET', s3_bucket, '') - conf.define('TC_AWS_LOADER_ROOT_PATH', 'root_path', '') - - context = Context(config=conf) - - s3_loader.load(context, IMAGE_PATH, callback) - - def should_load_from_s3(self, data): - image = data.args[0] - expect(image).to_equal(IMAGE_BYTES) - - class ValidatesBuckets(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - conf = Config() - conf.define('TC_AWS_ALLOWED_BUCKETS', [], '') - - context = Context(config=conf) - s3_loader.load(context, '/'.join([s3_bucket, IMAGE_PATH]), callback) - - def should_load_from_s3(self, data): - image = data.args[0] - expect(image).to_equal(None) - - class HandlesHttpLoader(Vows.Context): - - def topic(self): - conf = Config() - conf.define('TC_AWS_ENABLE_HTTP_LOADER', True, '') - - return Context(config=conf) - - @patch('thumbor.loaders.http_loader.load_sync') - def should_redirect_to_http(self, topic, load_sync_patch): - def callback(*args): - pass - - s3_loader.load(topic, 'http://foo.bar', callback) - expect(load_sync_patch.called).to_be_true() - - @mock_s3 - @patch('thumbor.loaders.http_loader.load_sync') - def should_not_redirect_to_http_if_not_prefixed_with_scheme(self, topic, load_sync_patch): - def callback(*args): - pass - - s3_loader.load(topic, 'foo.bar', callback) - expect(load_sync_patch.called).to_be_false() - - class HandleDataFuncLoader(Vows.Context): - - def topic(self): - return 3 - - def should_call_twice(self, topic): - - def callback(*args, **kwargs): - pass - - file_key = { - 'Error': 'Error', - 'ResponseMetadata': { - 'HTTPStatusCode': 502 - } - } - - self.call_count = 0 - def get(key, callback=None): - self.call_count += 1 - callback(file_key) - - mock_bucket_loader = MagicMock() - mock_bucket_loader.get = get - func = s3_loader.HandleDataFunc.as_func( - '/'.join([s3_bucket, IMAGE_PATH]), - callback=callback, - bucket_loader=mock_bucket_loader, - max_retry=topic - ) - - func(file_key) - expect(self.call_count).to_equal(3) diff --git a/vows/storage_vows.py b/vows/storage_vows.py deleted file mode 100644 index ef0727a..0000000 --- a/vows/storage_vows.py +++ /dev/null @@ -1,348 +0,0 @@ -#se!/usr/bin/python -# -*- coding: utf-8 -*- - -from pyvows import Vows, expect - -from datetime import datetime, timedelta -from dateutil.tz import tzutc - -from thumbor.context import Context, RequestParameters -from thumbor.config import Config -from fixtures.storage_fixture import IMAGE_URL, IMAGE_BYTES, get_server - -from boto.s3.connection import S3Connection -from moto import mock_s3_deprecated as mock_s3 - -from tc_aws.storages.s3_storage import Storage - -import logging -logging.getLogger('botocore').setLevel(logging.CRITICAL) - -s3_bucket = 'thumbor-images-test' - -@Vows.batch -class S3StorageVows(Vows.Context): - - class CanStoreImage(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - thumborId = IMAGE_URL % '1' - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) - storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) - storage.put(thumborId, IMAGE_BYTES) - storage.get(thumborId, callback=callback) - - def should_be_in_catalog(self, topic): - expect(topic.args[0]).not_to_be_null() - expect(topic.args[0]).not_to_be_an_error() - expect(topic.args[0]).to_equal(IMAGE_BYTES) - - class CanGetImage(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) - storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) - storage.put(IMAGE_URL % '2', IMAGE_BYTES) - storage.get(IMAGE_URL % '2', callback=callback) - - def should_not_be_null(self, topic): - expect(topic.args[0]).not_to_be_null() - expect(topic.args[0]).not_to_be_an_error() - - def should_have_proper_bytes(self, topic): - expect(topic.args[0]).to_equal(IMAGE_BYTES) - - class CanGetImageExistance(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) - storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) - storage.put(IMAGE_URL % '3', IMAGE_BYTES) - storage.exists(IMAGE_URL % '3', callback=callback) - - def should_exists(self, topic): - expect(topic.args[0]).to_equal(True) - - class CanGetImageInexistance(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) - storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) - storage.exists(IMAGE_URL % '9999', callback) - - def should_not_exists(self, topic): - expect(topic.args[0]).to_equal(False) - - class CanRemoveImage(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) - storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) - storage.put(IMAGE_URL % '4', IMAGE_BYTES) # 1: we put the image - storage.remove(IMAGE_URL % '4') # 2: we delete it - storage.exists(IMAGE_URL % '4', callback=callback) # 3: we check it exists - - def should_be_put_and_removed(self, topic): - expect(topic.args[0]).to_equal(False) # 4.1: assertion... - - class CanRemovethenPutImage(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) - storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) - storage.put(IMAGE_URL % '5', IMAGE_BYTES) # 1: we put the image - - def check_created(created): - expect(created).to_equal(True) # 2.1: assertion... - - def once_removed(rm): - - def check_created_2(exists): - expect(exists).to_equal(True) # 4.1: assertion... - - storage.put(IMAGE_URL % '5') # 5: we re-put it - storage.exists(IMAGE_URL % '5', callback=callback) #6: we check its existance again - - storage.exists(IMAGE_URL % '5', callback=check_created_2) #4: we check if the image exists - - storage.remove(IMAGE_URL % '5', callback=once_removed) # 3: we delete it - - storage.exists(IMAGE_URL % '5', callback=check_created) # 2: we check it exists - - def should_be_put_and_removed(self, topic): - expect(topic.args[0]).to_equal(True) - - class CanReturnPath(Vows.Context): - @mock_s3 - def topic(self): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) - storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) - return storage.resolve_original_photo_path("toto") - - def should_return_the_same(self, topic): - expect(topic).to_equal("toto") - - class HandlesStoragePrefix(Vows.Context): - @mock_s3 - def topic(self): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket, TC_AWS_STORAGE_ROOT_PATH='tata') - storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) - - return storage._normalize_path('toto') - - def should_return_the_same(self, topic): - expect(topic).to_equal("tata/toto") - - class ShouldNormalize(Vows.Context): - def topic(self): - config = Config(TC_AWS_STORAGE_ROOT_PATH='', TC_AWS_ROOT_IMAGE_NAME='root_image') - return Storage(Context(config=config)) - - def should_normalize_slash(self, topic): - expect(topic._normalize_path('/test')).to_equal('test') - expect(topic._normalize_path('/test/')).to_equal('test/root_image') - expect(topic._normalize_path('/test/image.png')).to_equal('test/image.png') - - class CryptoVows(Vows.Context): - class RaisesIfInvalidConfig(Vows.Context): - @Vows.capture_error - @mock_s3 - def topic(self): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket, STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True) - storage = Storage(Context(config=config, server=get_server(''))) - storage.put(IMAGE_URL % '9999', IMAGE_BYTES) - storage.put_crypto(IMAGE_URL % '9999') - - def should_be_an_error(self, topic): - expect(topic).to_be_an_error_like(RuntimeError) - expect(topic).to_have_an_error_message_of("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified") - - class GettingCryptoForANewImageReturnsNone(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket, STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True) - storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) - storage.get_crypto(IMAGE_URL % '9999', callback=callback) - - def should_be_null(self, topic): - expect(topic.args[0]).to_be_null() - - class DoesNotStoreIfConfigSaysNotTo(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) - storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) - storage.put(IMAGE_URL % '9998', IMAGE_BYTES) - storage.put_crypto(IMAGE_URL % '9998') - storage.get_crypto(IMAGE_URL % '9998', callback=callback) - - def should_be_null(self, topic): - expect(topic.args[0]).to_be_null() - - class CanStoreCrypto(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket, STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True) - storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) - storage.put(IMAGE_URL % '6', IMAGE_BYTES) - storage.put_crypto(IMAGE_URL % '6') - storage.get_crypto(IMAGE_URL % '6', callback=callback) - - def should_not_be_null(self, topic): - expect(topic.args[0]).not_to_be_null() - expect(topic.args[0]).not_to_be_an_error() - expect(topic.args[0]).to_equal('ACME-SEC') - - class DetectorVows(Vows.Context): - class CanStoreDetectorData(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) - storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) - storage.put(IMAGE_URL % '7', IMAGE_BYTES) - storage.put_detector_data(IMAGE_URL % '7', 'some-data') - storage.get_detector_data(IMAGE_URL % '7', callback=callback) - - def should_not_be_null(self, topic): - expect(topic.args[0]).not_to_be_null() - expect(topic.args[0]).not_to_be_an_error() - expect(topic.args[0]).to_equal('some-data') - - class ReturnsNoneIfNoDetectorData(Vows.Context): - @Vows.async_topic - @mock_s3 - def topic(self, callback): - self.conn = S3Connection() - self.conn.create_bucket(s3_bucket) - - config = Config(TC_AWS_STORAGE_BUCKET=s3_bucket) - storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) - storage.get_detector_data(IMAGE_URL % '9999', callback=callback) - - def should_not_be_null(self, topic): - expect(topic.args[0]).to_be_null() - - class WebPVows(Vows.Context): - class HasConfigRequest(Vows.Context): - def topic(self): - config = Config(AUTO_WEBP=True) - context = Context(config=config) - context.request = RequestParameters(accepts_webp=True) - return Storage(context) - - def should_be_webp(self, topic): - expect(topic.is_auto_webp).to_be_true() - - class HasConfigNoRequest(Vows.Context): - def topic(self): - config = Config(AUTO_WEBP=True) - context = Context(config=config) - return Storage(context) - - def should_be_webp(self, topic): - expect(topic.is_auto_webp).to_be_false() - - class HasConfigRequestDoesNotAccept(Vows.Context): - def topic(self): - config = Config(AUTO_WEBP=True) - context = Context(config=config) - context.request = RequestParameters(accepts_webp=False) - return Storage(context) - - def should_be_webp(self, topic): - expect(topic.is_auto_webp).to_be_false() - - class HasNotConfig(Vows.Context): - def topic(self): - config = Config(AUTO_WEBP=False) - context = Context(config=config) - context.request = RequestParameters(accepts_webp=True) - return Storage(context) - - def should_be_webp(self, topic): - expect(topic.is_auto_webp).to_be_false() - - class ExpiredVows(Vows.Context): - class WhenExpiredEnabled(Vows.Context): - def topic(self): - return Storage(Context(config=Config(STORAGE_EXPIRATION_SECONDS=3600))) - - def should_check_invalid_key(self, topic): - expect(topic.is_expired(None)).to_be_true() - expect(topic.is_expired(False)).to_be_true() - expect(topic.is_expired(dict())).to_be_true() - expect(topic.is_expired({'Error': ''})).to_be_true() - - def should_tell_when_not_expired(self, topic): - key = { - 'LastModified': datetime.now(tzutc()), - 'Body': 'foobar', - } - expect(topic.is_expired(key)).to_be_false() - - def should_tell_when_expired(self, topic): - key = { - 'LastModified': (datetime.now(tzutc()) - timedelta(seconds=3601)), - 'Body': 'foobar', - } - expect(topic.is_expired(key)).to_be_true() - - class WhenExpiredDisabled(Vows.Context): - def topic(self): - return Storage(Context(config=Config(STORAGE_EXPIRATION_SECONDS=0))) - - def should_not_tell_when_expired(self, topic): - key = { - 'LastModified': (datetime.now(tzutc()) - timedelta(seconds=3601)), - 'Body': 'foobar', - } - expect(topic.is_expired(key)).to_be_false()