Add writable flag to meta stores (#256)

* Add writable flag to meta store classes

* Always compute metadata

* 🐛

* Don't assert always true statement

* Handle writability in meta stores

* Error naming and specific try-catch

* Add unwritable TC driver test

* Also require writable for database creation

* Add non-writable error handler

* Add flask debug exception tests

* 💄

* Monkeypatch meta_store._WRITABLE

* Actually use the monkeypatch...

Co-authored-by: Dion Häfner <[email protected]>

* Use the contexts

Co-authored-by: Nicklas Boserup <[email protected]>
Co-authored-by: Dion Häfner <[email protected]>

terracotta/drivers/base_classes.py

@@ -10,12 +10,25 @@
 from typing import (Any, Callable, Dict, List, Mapping, Optional, Sequence,
                     Tuple, TypeVar, Union)
 
+from terracotta import exceptions
+
 KeysType = Mapping[str, str]
 MultiValueKeysType = Mapping[str, Union[str, List[str]]]
 Number = TypeVar('Number', int, float)
 T = TypeVar('T')
 
 
+def requires_writable(fun: Callable[..., T]) -> Callable[..., T]:
+    @functools.wraps(fun)
+    def inner(self: MetaStore, *args: Any, **kwargs: Any) -> T:
+        if self._WRITABLE:
+            return fun(self, *args, **kwargs)
+        else:
+            raise exceptions.DatabaseNotWritableError("Database not writable")
+
+    return inner
+
+
 def requires_connection(
     fun: Callable[..., T] = None, *,
     verify: bool = True
@@ -38,6 +51,7 @@ class MetaStore(ABC):
     Defines a common interface for all metadata backends.
     """
     _RESERVED_KEYS = ('limit', 'page')
+    _WRITABLE: bool = True
 
     @property
     @abstractmethod

terracotta/drivers/relational_meta_store.py

@@ -21,7 +21,8 @@
 from terracotta import exceptions
 from terracotta.drivers.base_classes import (KeysType, MetaStore,
                                              MultiValueKeysType,
-                                             requires_connection)
+                                             requires_connection,
+                                             requires_writable)
 from terracotta.profile import trace
 
 _ERROR_ON_CONNECT = (
@@ -181,6 +182,7 @@ def db_version(self) -> str:
         version = self.connection.execute(stmt).scalar()
         return version
 
+    @requires_writable
     @convert_exceptions('Could not create database')
     def create(self, keys: Sequence[str], key_descriptions: Mapping[str, str] = None) -> None:
         """Create and initialize database with empty tables.
@@ -349,6 +351,7 @@ def get_metadata(self, keys: KeysType) -> Optional[Dict[str, Any]]:
         return self._decode_data(encoded_data)
 
     @trace('insert')
+    @requires_writable
     @requires_connection
     @convert_exceptions('Could not write to database')
     def insert(
@@ -383,6 +386,7 @@ def insert(
             )
 
     @trace('delete')
+    @requires_writable
     @requires_connection
     @convert_exceptions('Could not write to database')
     def delete(self, keys: KeysType) -> None:

terracotta/drivers/sqlite_remote_meta_store.py

@@ -11,7 +11,7 @@
 import time
 import urllib.parse as urlparse
 from pathlib import Path
-from typing import Any, Iterator, Union
+from typing import Iterator, Union
 
 from terracotta import exceptions, get_settings
 from terracotta.drivers.sqlite_meta_store import SQLiteMetaStore
@@ -69,10 +69,12 @@ class RemoteSQLiteMetaStore(SQLiteMetaStore):
     Warning:
 
         This driver is read-only. Any attempts to use the create, insert, or delete methods
-        will throw a NotImplementedError.
+        will throw a DatabaseNotWritableError.
 
     """
 
+    _WRITABLE: bool = False
+
     def __init__(self, remote_path: Union[str, Path]) -> None:
         """Initialize the RemoteSQLiteDriver.
 
@@ -132,15 +134,6 @@ def _connection_callback(self) -> None:
         self._update_db(self._remote_path, self._local_path)
         super()._connection_callback()
 
-    def create(self, *args: Any, **kwargs: Any) -> None:
-        raise NotImplementedError('Remote SQLite databases are read-only')
-
-    def insert(self, *args: Any, **kwargs: Any) -> None:
-        raise NotImplementedError('Remote SQLite databases are read-only')
-
-    def delete(self, *args: Any, **kwargs: Any) -> None:
-        raise NotImplementedError('Remote SQLite databases are read-only')
-
     def __del__(self) -> None:
         """Clean up temporary database upon exit"""
         self.__rm(self._local_path)

terracotta/drivers/terracotta_driver.py

@@ -5,8 +5,8 @@
 
 import contextlib
 from collections import OrderedDict
-from typing import (Any, Collection, Dict, List, Mapping, Optional, Sequence, Tuple, TypeVar,
-                    Union)
+from typing import (Any, Collection, Dict, List, Mapping,
+                    Optional, Sequence, Tuple, TypeVar, Union)
 
 import terracotta
 from terracotta import exceptions
@@ -169,7 +169,13 @@ def get_metadata(self, keys: ExtendedKeysType) -> Dict[str, Any]:
 
             path = squeeze(dataset.values())
             metadata = self.compute_metadata(path, max_shape=self.LAZY_LOADING_MAX_SHAPE)
-            self.insert(keys, path, metadata=metadata)
+
+            try:
+                self.insert(keys, path, metadata=metadata)
+            except exceptions.DatabaseNotWritableError as exc:
+                raise exceptions.DatabaseNotWritableError(
+                    "Lazy loading requires a writable database"
+                ) from exc
 
             # ensure standardized/consistent output (types and floating point precision)
             metadata = self.meta_store.get_metadata(keys)

terracotta/exceptions.py

@@ -24,5 +24,9 @@ class InvalidDatabaseError(Exception):
     pass
 
 
+class DatabaseNotWritableError(Exception):
+    pass
+
+
 class PerformanceWarning(UserWarning):
     pass

terracotta/server/flask_api.py

@@ -63,6 +63,14 @@ def handle_dataset_not_found_error(exc: Exception) -> Any:
 
     register_error_handler(exceptions.DatasetNotFoundError, handle_dataset_not_found_error)
 
+    def handle_database_not_writable_error(exc: Exception) -> Any:
+        # database not writable -> 403
+        if current_app.debug:
+            raise exc
+        return _abort(403, str(exc))
+
+    register_error_handler(exceptions.DatabaseNotWritableError, handle_database_not_writable_error)
+
     def handle_marshmallow_validation_error(exc: Exception) -> Any:
         # wrong query arguments -> 400
         if current_app.debug:

tests/conftest.py

@@ -353,6 +353,22 @@ def use_testdb(testdb, monkeypatch):
     terracotta.update_settings(DRIVER_PATH=str(testdb))
 
 
+@pytest.fixture()
+def use_non_writable_testdb(testdb, monkeypatch, raster_file):
+    import terracotta
+    terracotta.update_settings(DRIVER_PATH=str(testdb))
+
+    driver = terracotta.get_driver(testdb)
+    with driver.connect():
+        driver.insert(('first', 'second', 'third'), str(raster_file), skip_metadata=True)
+
+    with monkeypatch.context() as m:
+        m.setattr(driver.meta_store, "_WRITABLE", False)
+        yield
+
+    driver.delete(('first', 'second', 'third'))
+
+
 def run_test_server(driver_path, port):
     from terracotta import update_settings
     update_settings(DRIVER_PATH=driver_path)

tests/drivers/test_raster_drivers.py

@@ -5,10 +5,28 @@
 
 import numpy as np
 
+from terracotta import exceptions
+
 DRIVERS = ['sqlite', 'mysql']
 METADATA_KEYS = ('bounds', 'range', 'mean', 'stdev', 'percentiles', 'metadata')
 
 
+@pytest.fixture
+def dynamic_non_writable_db(monkeypatch, driver_path, provider, raster_file):
+    from terracotta import drivers
+    db = drivers.get_driver(driver_path, provider=provider)
+    keys = ('some', 'keynames')
+
+    db.create(keys)
+    db.insert(['some', 'value'], str(raster_file), skip_metadata=True)
+
+    with monkeypatch.context() as m:
+        m.setattr(db.meta_store, "_WRITABLE", False)
+        yield
+
+    db.delete(['some', 'value'])
+
+
 @pytest.mark.parametrize('provider', DRIVERS)
 def test_insertion_and_retrieval(driver_path, provider, raster_file):
     from terracotta import drivers
@@ -144,6 +162,15 @@ def test_lazy_loading(driver_path, provider, raster_file):
     assert all(np.all(data1[k] == data2[k]) for k in data1.keys())
 
 
+@pytest.mark.parametrize('provider', DRIVERS)
+def test_non_writable_lazy_loading(driver_path, provider, dynamic_non_writable_db):
+    from terracotta import drivers
+    db = drivers.get_driver(driver_path, provider=provider)
+
+    with pytest.raises(exceptions.DatabaseNotWritableError):
+        db.get_metadata(['some', 'value'])
+
+
 @pytest.mark.parametrize('provider', DRIVERS)
 def test_precomputed_metadata(driver_path, provider, raster_file):
     from terracotta import drivers

tests/drivers/test_sqlite_remote.py

@@ -11,6 +11,8 @@
 
 import pytest
 
+from terracotta import exceptions
+
 boto3 = pytest.importorskip('boto3')
 moto = pytest.importorskip('moto')
 
@@ -142,13 +144,13 @@ def test_immutability(s3_db_factory, raster_file):
 
     driver = get_driver(dbpath)
 
-    with pytest.raises(NotImplementedError):
+    with pytest.raises(exceptions.DatabaseNotWritableError):
         driver.create(keys)
 
-    with pytest.raises(NotImplementedError):
+    with pytest.raises(exceptions.DatabaseNotWritableError):
         driver.insert(('some', 'value'), str(raster_file))
 
-    with pytest.raises(NotImplementedError):
+    with pytest.raises(exceptions.DatabaseNotWritableError):
         driver.delete(('some', 'value'))
 
 

tests/server/test_flask_api.py

@@ -21,6 +21,18 @@ def client(flask_app):
         yield client
 
 
+@pytest.fixture(scope='module')
+def debug_flask_app():
+    from terracotta.server import create_app
+    return create_app(debug=True)
+
+
+@pytest.fixture(scope='module')
+def debug_client(debug_flask_app):
+    with debug_flask_app.test_client() as client:
+        yield client
+
+
 def test_get_keys(client, use_testdb):
     rv = client.get('/keys')
 
@@ -39,6 +51,40 @@ def test_get_metadata(client, use_testdb):
     assert ['extra_data'] == json.loads(rv.data)['metadata']
 
 
+def test_get_metadata_lazily_nonwritable_db(client, use_non_writable_testdb):
+    rv = client.get('/metadata/first/second/third')
+    assert rv.status_code == 403
+
+
+def test_debug_errors(debug_client, use_non_writable_testdb, raster_file_xyz):
+    from terracotta import exceptions
+    import marshmallow
+
+    with pytest.raises(exceptions.DatabaseNotWritableError):
+        debug_client.get('/metadata/first/second/third')
+
+    with pytest.raises(exceptions.DatasetNotFoundError):
+        debug_client.get('/metadata/NONEXISTING/KEYS/YO')
+
+    with pytest.raises(exceptions.InvalidKeyError):
+        debug_client.get('/metadata/ONLYONEKEY')
+
+    x, y, z = raster_file_xyz
+
+    with pytest.raises(exceptions.InvalidArgumentsError):
+        debug_client.get(
+            f'/compute/first/second/third/{z}/{x}/{y}.png'
+            '?expression=v1*v2&v1=val22&v2=val23'
+            '&stretch_range=[10000,0]'
+        )
+
+    with pytest.raises(marshmallow.ValidationError):
+        debug_client.get(
+            f'/compute/first/second/third/{z}/{x}/{y}.png'
+            '?stretch_range=[10000,0]'
+        )
+
+
 def test_get_metadata_nonexisting(client, use_testdb):
     rv = client.get('/metadata/val11/x/NONEXISTING/')
     assert rv.status_code == 404