diff --git a/terracotta/drivers/base_classes.py b/terracotta/drivers/base_classes.py index bae3f206..a4c99f29 100644 --- a/terracotta/drivers/base_classes.py +++ b/terracotta/drivers/base_classes.py @@ -10,12 +10,25 @@ from typing import (Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, TypeVar, Union) +from terracotta import exceptions + KeysType = Mapping[str, str] MultiValueKeysType = Mapping[str, Union[str, List[str]]] Number = TypeVar('Number', int, float) T = TypeVar('T') +def requires_writable(fun: Callable[..., T]) -> Callable[..., T]: + @functools.wraps(fun) + def inner(self: MetaStore, *args: Any, **kwargs: Any) -> T: + if self._WRITABLE: + return fun(self, *args, **kwargs) + else: + raise exceptions.DatabaseNotWritableError("Database not writable") + + return inner + + def requires_connection( fun: Callable[..., T] = None, *, verify: bool = True @@ -38,6 +51,7 @@ class MetaStore(ABC): Defines a common interface for all metadata backends. """ _RESERVED_KEYS = ('limit', 'page') + _WRITABLE: bool = True @property @abstractmethod diff --git a/terracotta/drivers/relational_meta_store.py b/terracotta/drivers/relational_meta_store.py index 2ccdf45f..eb247bf3 100644 --- a/terracotta/drivers/relational_meta_store.py +++ b/terracotta/drivers/relational_meta_store.py @@ -21,7 +21,8 @@ from terracotta import exceptions from terracotta.drivers.base_classes import (KeysType, MetaStore, MultiValueKeysType, - requires_connection) + requires_connection, + requires_writable) from terracotta.profile import trace _ERROR_ON_CONNECT = ( @@ -181,6 +182,7 @@ def db_version(self) -> str: version = self.connection.execute(stmt).scalar() return version + @requires_writable @convert_exceptions('Could not create database') def create(self, keys: Sequence[str], key_descriptions: Mapping[str, str] = None) -> None: """Create and initialize database with empty tables. @@ -349,6 +351,7 @@ def get_metadata(self, keys: KeysType) -> Optional[Dict[str, Any]]: return self._decode_data(encoded_data) @trace('insert') + @requires_writable @requires_connection @convert_exceptions('Could not write to database') def insert( @@ -383,6 +386,7 @@ def insert( ) @trace('delete') + @requires_writable @requires_connection @convert_exceptions('Could not write to database') def delete(self, keys: KeysType) -> None: diff --git a/terracotta/drivers/sqlite_remote_meta_store.py b/terracotta/drivers/sqlite_remote_meta_store.py index e40968ad..d2c999ce 100644 --- a/terracotta/drivers/sqlite_remote_meta_store.py +++ b/terracotta/drivers/sqlite_remote_meta_store.py @@ -11,7 +11,7 @@ import time import urllib.parse as urlparse from pathlib import Path -from typing import Any, Iterator, Union +from typing import Iterator, Union from terracotta import exceptions, get_settings from terracotta.drivers.sqlite_meta_store import SQLiteMetaStore @@ -69,10 +69,12 @@ class RemoteSQLiteMetaStore(SQLiteMetaStore): Warning: This driver is read-only. Any attempts to use the create, insert, or delete methods - will throw a NotImplementedError. + will throw a DatabaseNotWritableError. """ + _WRITABLE: bool = False + def __init__(self, remote_path: Union[str, Path]) -> None: """Initialize the RemoteSQLiteDriver. @@ -132,15 +134,6 @@ def _connection_callback(self) -> None: self._update_db(self._remote_path, self._local_path) super()._connection_callback() - def create(self, *args: Any, **kwargs: Any) -> None: - raise NotImplementedError('Remote SQLite databases are read-only') - - def insert(self, *args: Any, **kwargs: Any) -> None: - raise NotImplementedError('Remote SQLite databases are read-only') - - def delete(self, *args: Any, **kwargs: Any) -> None: - raise NotImplementedError('Remote SQLite databases are read-only') - def __del__(self) -> None: """Clean up temporary database upon exit""" self.__rm(self._local_path) diff --git a/terracotta/drivers/terracotta_driver.py b/terracotta/drivers/terracotta_driver.py index e0ac77d1..2fac7588 100644 --- a/terracotta/drivers/terracotta_driver.py +++ b/terracotta/drivers/terracotta_driver.py @@ -5,8 +5,8 @@ import contextlib from collections import OrderedDict -from typing import (Any, Collection, Dict, List, Mapping, Optional, Sequence, Tuple, TypeVar, - Union) +from typing import (Any, Collection, Dict, List, Mapping, + Optional, Sequence, Tuple, TypeVar, Union) import terracotta from terracotta import exceptions @@ -169,7 +169,13 @@ def get_metadata(self, keys: ExtendedKeysType) -> Dict[str, Any]: path = squeeze(dataset.values()) metadata = self.compute_metadata(path, max_shape=self.LAZY_LOADING_MAX_SHAPE) - self.insert(keys, path, metadata=metadata) + + try: + self.insert(keys, path, metadata=metadata) + except exceptions.DatabaseNotWritableError as exc: + raise exceptions.DatabaseNotWritableError( + "Lazy loading requires a writable database" + ) from exc # ensure standardized/consistent output (types and floating point precision) metadata = self.meta_store.get_metadata(keys) diff --git a/terracotta/exceptions.py b/terracotta/exceptions.py index b6d904b2..6ba68e74 100644 --- a/terracotta/exceptions.py +++ b/terracotta/exceptions.py @@ -24,5 +24,9 @@ class InvalidDatabaseError(Exception): pass +class DatabaseNotWritableError(Exception): + pass + + class PerformanceWarning(UserWarning): pass diff --git a/terracotta/server/flask_api.py b/terracotta/server/flask_api.py index c9976c84..4801925e 100644 --- a/terracotta/server/flask_api.py +++ b/terracotta/server/flask_api.py @@ -63,6 +63,14 @@ def handle_dataset_not_found_error(exc: Exception) -> Any: register_error_handler(exceptions.DatasetNotFoundError, handle_dataset_not_found_error) + def handle_database_not_writable_error(exc: Exception) -> Any: + # database not writable -> 403 + if current_app.debug: + raise exc + return _abort(403, str(exc)) + + register_error_handler(exceptions.DatabaseNotWritableError, handle_database_not_writable_error) + def handle_marshmallow_validation_error(exc: Exception) -> Any: # wrong query arguments -> 400 if current_app.debug: diff --git a/tests/conftest.py b/tests/conftest.py index 44a85ffe..950bfa8e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -344,6 +344,22 @@ def use_testdb(testdb, monkeypatch): terracotta.update_settings(DRIVER_PATH=str(testdb)) +@pytest.fixture() +def use_non_writable_testdb(testdb, monkeypatch, raster_file): + import terracotta + terracotta.update_settings(DRIVER_PATH=str(testdb)) + + driver = terracotta.get_driver(testdb) + with driver.connect(): + driver.insert(('first', 'second', 'third'), str(raster_file), skip_metadata=True) + + with monkeypatch.context() as m: + m.setattr(driver.meta_store, "_WRITABLE", False) + yield + + driver.delete(('first', 'second', 'third')) + + def run_test_server(driver_path, port): from terracotta import update_settings update_settings(DRIVER_PATH=driver_path) diff --git a/tests/drivers/test_raster_drivers.py b/tests/drivers/test_raster_drivers.py index 7954cf07..5eb4d859 100644 --- a/tests/drivers/test_raster_drivers.py +++ b/tests/drivers/test_raster_drivers.py @@ -5,10 +5,28 @@ import numpy as np +from terracotta import exceptions + DRIVERS = ['sqlite', 'mysql'] METADATA_KEYS = ('bounds', 'range', 'mean', 'stdev', 'percentiles', 'metadata') +@pytest.fixture +def dynamic_non_writable_db(monkeypatch, driver_path, provider, raster_file): + from terracotta import drivers + db = drivers.get_driver(driver_path, provider=provider) + keys = ('some', 'keynames') + + db.create(keys) + db.insert(['some', 'value'], str(raster_file), skip_metadata=True) + + with monkeypatch.context() as m: + m.setattr(db.meta_store, "_WRITABLE", False) + yield + + db.delete(['some', 'value']) + + @pytest.mark.parametrize('provider', DRIVERS) def test_insertion_and_retrieval(driver_path, provider, raster_file): from terracotta import drivers @@ -144,6 +162,15 @@ def test_lazy_loading(driver_path, provider, raster_file): assert all(np.all(data1[k] == data2[k]) for k in data1.keys()) +@pytest.mark.parametrize('provider', DRIVERS) +def test_non_writable_lazy_loading(driver_path, provider, dynamic_non_writable_db): + from terracotta import drivers + db = drivers.get_driver(driver_path, provider=provider) + + with pytest.raises(exceptions.DatabaseNotWritableError): + db.get_metadata(['some', 'value']) + + @pytest.mark.parametrize('provider', DRIVERS) def test_precomputed_metadata(driver_path, provider, raster_file): from terracotta import drivers diff --git a/tests/drivers/test_sqlite_remote.py b/tests/drivers/test_sqlite_remote.py index d4852617..395ab541 100644 --- a/tests/drivers/test_sqlite_remote.py +++ b/tests/drivers/test_sqlite_remote.py @@ -11,6 +11,8 @@ import pytest +from terracotta import exceptions + boto3 = pytest.importorskip('boto3') moto = pytest.importorskip('moto') @@ -142,13 +144,13 @@ def test_immutability(s3_db_factory, raster_file): driver = get_driver(dbpath) - with pytest.raises(NotImplementedError): + with pytest.raises(exceptions.DatabaseNotWritableError): driver.create(keys) - with pytest.raises(NotImplementedError): + with pytest.raises(exceptions.DatabaseNotWritableError): driver.insert(('some', 'value'), str(raster_file)) - with pytest.raises(NotImplementedError): + with pytest.raises(exceptions.DatabaseNotWritableError): driver.delete(('some', 'value')) diff --git a/tests/server/test_flask_api.py b/tests/server/test_flask_api.py index 0ee0f9b6..430f62e6 100644 --- a/tests/server/test_flask_api.py +++ b/tests/server/test_flask_api.py @@ -21,6 +21,18 @@ def client(flask_app): yield client +@pytest.fixture(scope='module') +def debug_flask_app(): + from terracotta.server import create_app + return create_app(debug=True) + + +@pytest.fixture(scope='module') +def debug_client(debug_flask_app): + with debug_flask_app.test_client() as client: + yield client + + def test_get_keys(client, use_testdb): rv = client.get('/keys') @@ -39,6 +51,40 @@ def test_get_metadata(client, use_testdb): assert ['extra_data'] == json.loads(rv.data)['metadata'] +def test_get_metadata_lazily_nonwritable_db(client, use_non_writable_testdb): + rv = client.get('/metadata/first/second/third') + assert rv.status_code == 403 + + +def test_debug_errors(debug_client, use_non_writable_testdb, raster_file_xyz): + from terracotta import exceptions + import marshmallow + + with pytest.raises(exceptions.DatabaseNotWritableError): + debug_client.get('/metadata/first/second/third') + + with pytest.raises(exceptions.DatasetNotFoundError): + debug_client.get('/metadata/NONEXISTING/KEYS/YO') + + with pytest.raises(exceptions.InvalidKeyError): + debug_client.get('/metadata/ONLYONEKEY') + + x, y, z = raster_file_xyz + + with pytest.raises(exceptions.InvalidArgumentsError): + debug_client.get( + f'/compute/first/second/third/{z}/{x}/{y}.png' + '?expression=v1*v2&v1=val22&v2=val23' + '&stretch_range=[10000,0]' + ) + + with pytest.raises(marshmallow.ValidationError): + debug_client.get( + f'/compute/first/second/third/{z}/{x}/{y}.png' + '?stretch_range=[10000,0]' + ) + + def test_get_metadata_nonexisting(client, use_testdb): rv = client.get('/metadata/val11/x/NONEXISTING/') assert rv.status_code == 404