Add writable flag to meta stores

terracotta/drivers/base_classes.py

@@ -10,12 +10,25 @@
 from typing import (Any, Callable, Dict, List, Mapping, Optional, Sequence,
                     Tuple, TypeVar, Union)
 
+from terracotta import exceptions
+
 KeysType = Mapping[str, str]
 MultiValueKeysType = Mapping[str, Union[str, List[str]]]
 Number = TypeVar('Number', int, float)
 T = TypeVar('T')
 
 
+def requires_writable(fun: Callable[..., T]) -> Callable[..., T]:
+    @functools.wraps(fun)
+    def inner(self: MetaStore, *args: Any, **kwargs: Any) -> T:
+        if self._WRITABLE:
+            return fun(self, *args, **kwargs)
+        else:
+            raise exceptions.DatabaseNotWritableError("Database not writable")
+
+    return inner
+
+
 def requires_connection(
     fun: Callable[..., T] = None, *,
     verify: bool = True
@@ -38,6 +51,7 @@ class MetaStore(ABC):
     Defines a common interface for all metadata backends.
     """
     _RESERVED_KEYS = ('limit', 'page')
+    _WRITABLE: bool = True
 
     @property
     @abstractmethod

terracotta/drivers/relational_meta_store.py

@@ -21,7 +21,8 @@
 from terracotta import exceptions
 from terracotta.drivers.base_classes import (KeysType, MetaStore,
                                              MultiValueKeysType,
-                                             requires_connection)
+                                             requires_connection,
+                                             requires_writable)
 from terracotta.profile import trace
 
 _ERROR_ON_CONNECT = (
@@ -181,6 +182,7 @@ def db_version(self) -> str:
         version = self.connection.execute(stmt).scalar()
         return version
 
+    @requires_writable
     @convert_exceptions('Could not create database')
     def create(self, keys: Sequence[str], key_descriptions: Mapping[str, str] = None) -> None:
         """Create and initialize database with empty tables.
@@ -349,6 +351,7 @@ def get_metadata(self, keys: KeysType) -> Optional[Dict[str, Any]]:
         return self._decode_data(encoded_data)
 
     @trace('insert')
+    @requires_writable
     @requires_connection
     @convert_exceptions('Could not write to database')
     def insert(
@@ -383,6 +386,7 @@ def insert(
             )
 
     @trace('delete')
+    @requires_writable
     @requires_connection
     @convert_exceptions('Could not write to database')
     def delete(self, keys: KeysType) -> None:

terracotta/drivers/sqlite_remote_meta_store.py

@@ -11,7 +11,7 @@
 import time
 import urllib.parse as urlparse
 from pathlib import Path
-from typing import Any, Iterator, Union
+from typing import Iterator, Union
 
 from terracotta import exceptions, get_settings
 from terracotta.drivers.sqlite_meta_store import SQLiteMetaStore
@@ -69,10 +69,12 @@ class RemoteSQLiteMetaStore(SQLiteMetaStore):
     Warning:
 
         This driver is read-only. Any attempts to use the create, insert, or delete methods
-        will throw a NotImplementedError.
+        will throw a DatabaseNotWritableError.
 
     """
 
+    _WRITABLE: bool = False
+
     def __init__(self, remote_path: Union[str, Path]) -> None:
         """Initialize the RemoteSQLiteDriver.
 
@@ -132,15 +134,6 @@ def _connection_callback(self) -> None:
         self._update_db(self._remote_path, self._local_path)
         super()._connection_callback()
 
-    def create(self, *args: Any, **kwargs: Any) -> None:
-        raise NotImplementedError('Remote SQLite databases are read-only')
-
-    def insert(self, *args: Any, **kwargs: Any) -> None:
-        raise NotImplementedError('Remote SQLite databases are read-only')
-
-    def delete(self, *args: Any, **kwargs: Any) -> None:
-        raise NotImplementedError('Remote SQLite databases are read-only')
-
     def __del__(self) -> None:
         """Clean up temporary database upon exit"""
         self.__rm(self._local_path)

terracotta/drivers/terracotta_driver.py

@@ -5,8 +5,8 @@
 
 import contextlib
 from collections import OrderedDict
-from typing import (Any, Collection, Dict, List, Mapping, Optional, Sequence, Tuple, TypeVar,
-                    Union)
+from typing import (Any, Collection, Dict, List, Mapping,
+                    Optional, Sequence, Tuple, TypeVar, Union)
 
 import terracotta
 from terracotta import exceptions
@@ -169,7 +169,13 @@ def get_metadata(self, keys: ExtendedKeysType) -> Dict[str, Any]:
 
             path = squeeze(dataset.values())
             metadata = self.compute_metadata(path, max_shape=self.LAZY_LOADING_MAX_SHAPE)
-            self.insert(keys, path, metadata=metadata)
+
+            try:
+                self.insert(keys, path, metadata=metadata)
+            except exceptions.DatabaseNotWritableError as exc:
+                raise exceptions.DatabaseNotWritableError(
+                    "Lazy loading requires a writable database"
+                ) from exc
 
             # ensure standardized/consistent output (types and floating point precision)
             metadata = self.meta_store.get_metadata(keys)

terracotta/exceptions.py

@@ -24,5 +24,9 @@ class InvalidDatabaseError(Exception):
     pass
 
 
+class DatabaseNotWritableError(Exception):
+    pass
+
+
 class PerformanceWarning(UserWarning):
     pass

terracotta/server/flask_api.py

@@ -63,6 +63,14 @@ def handle_dataset_not_found_error(exc: Exception) -> Any:
 
     register_error_handler(exceptions.DatasetNotFoundError, handle_dataset_not_found_error)
 
+    def handle_database_not_writable_error(exc: Exception) -> Any:
+        # database not writable -> 403
+        if current_app.debug:
+            raise exc
+        return _abort(403, str(exc))
+
+    register_error_handler(exceptions.DatabaseNotWritableError, handle_database_not_writable_error)
+
     def handle_marshmallow_validation_error(exc: Exception) -> Any:
         # wrong query arguments -> 400
         if current_app.debug:

tests/conftest.py

@@ -3,6 +3,9 @@
 import shapely.geometry  # noqa: F401
 
 import os
+from pathlib import Path
+import tempfile
+import uuid
 import multiprocessing
 import time
 from functools import partial
@@ -12,6 +15,8 @@
 import numpy as np
 import rasterio
 
+boto3 = pytest.importorskip('boto3')
+
 
 def pytest_configure(config):
     os.environ['TC_TESTING'] = '1'
@@ -344,6 +349,46 @@ def use_testdb(testdb, monkeypatch):
     terracotta.update_settings(DRIVER_PATH=str(testdb))
 
 
+@pytest.fixture()
+def s3_db_factory(tmpdir):
+    bucketname = str(uuid.uuid4())
+
+    def _s3_db_factory(keys, datasets=None, skip_metadata=False):
+        from terracotta import get_driver
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            dbfile = Path(tmpdir) / 'tc.sqlite'
+            driver = get_driver(dbfile)
+            driver.create(keys)
+
+            if datasets:
+                for keys, path in datasets.items():
+                    driver.insert(keys, path, skip_metadata=skip_metadata)
+
+            with open(dbfile, 'rb') as f:
+                db_bytes = f.read()
+
+        conn = boto3.resource('s3')
+        conn.create_bucket(Bucket=bucketname)
+
+        s3 = boto3.client('s3')
+        s3.put_object(Bucket=bucketname, Key='tc.sqlite', Body=db_bytes)
+
+        return f's3://{bucketname}/tc.sqlite'
+
+    return _s3_db_factory
+
+
+@pytest.fixture()
+def mock_aws_env(monkeypatch):
+    with monkeypatch.context() as m:
+        m.setenv('AWS_DEFAULT_REGION', 'us-east-1')
+        m.setenv('AWS_ACCESS_KEY_ID', 'FakeKey')
+        m.setenv('AWS_SECRET_ACCESS_KEY', 'FakeSecretKey')
+        m.setenv('AWS_SESSION_TOKEN', 'FakeSessionToken')
+        yield
+
+
 def run_test_server(driver_path, port):
     from terracotta import update_settings
     update_settings(DRIVER_PATH=driver_path)

tests/drivers/test_raster_drivers.py

@@ -5,6 +5,8 @@
 
 import numpy as np
 
+from terracotta import exceptions
+
 DRIVERS = ['sqlite', 'mysql']
 METADATA_KEYS = ('bounds', 'range', 'mean', 'stdev', 'percentiles', 'metadata')
 
@@ -144,6 +146,22 @@ def test_lazy_loading(driver_path, provider, raster_file):
     assert all(np.all(data1[k] == data2[k]) for k in data1.keys())
 
 
+@pytest.mark.parametrize('provider', DRIVERS)
+def test_non_writable_lazy_loading(driver_path, provider, raster_file):
+    from terracotta import drivers
+    db = drivers.get_driver(driver_path, provider=provider)
+    keys = ('some', 'keynames')
+
+    db.create(keys)
+    db.insert(['some', 'value'], str(raster_file), skip_metadata=True)
+
+    # Manually set the meta store to un-writable
+    db.meta_store._WRITABLE = False
+
+    with pytest.raises(exceptions.DatabaseNotWritableError):
+        db.get_metadata(['some', 'value'])
+
+
 @pytest.mark.parametrize('provider', DRIVERS)
 def test_precomputed_metadata(driver_path, provider, raster_file):
     from terracotta import drivers

tests/drivers/test_sqlite_remote.py

@@ -4,25 +4,13 @@
 """
 
 import os
-import tempfile
 import time
-import uuid
-from pathlib import Path
 
 import pytest
 
-boto3 = pytest.importorskip('boto3')
-moto = pytest.importorskip('moto')
-
+from terracotta import exceptions
 
-@pytest.fixture(autouse=True)
-def mock_aws_env(monkeypatch):
-    with monkeypatch.context() as m:
-        m.setenv('AWS_DEFAULT_REGION', 'us-east-1')
-        m.setenv('AWS_ACCESS_KEY_ID', 'FakeKey')
-        m.setenv('AWS_SECRET_ACCESS_KEY', 'FakeSecretKey')
-        m.setenv('AWS_SESSION_TOKEN', 'FakeSessionToken')
-        yield
+moto = pytest.importorskip('moto')
 
 
 class Timer:
@@ -39,38 +27,8 @@ def tick(self):
         self.time += 1
 
 
-@pytest.fixture()
-def s3_db_factory(tmpdir):
-    bucketname = str(uuid.uuid4())
-
-    def _s3_db_factory(keys, datasets=None):
-        from terracotta import get_driver
-
-        with tempfile.TemporaryDirectory() as tmpdir:
-            dbfile = Path(tmpdir) / 'tc.sqlite'
-            driver = get_driver(dbfile)
-            driver.create(keys)
-
-            if datasets:
-                for keys, path in datasets.items():
-                    driver.insert(keys, path)
-
-            with open(dbfile, 'rb') as f:
-                db_bytes = f.read()
-
-        conn = boto3.resource('s3')
-        conn.create_bucket(Bucket=bucketname)
-
-        s3 = boto3.client('s3')
-        s3.put_object(Bucket=bucketname, Key='tc.sqlite', Body=db_bytes)
-
-        return f's3://{bucketname}/tc.sqlite'
-
-    return _s3_db_factory
-
-
 @moto.mock_s3
-def test_remote_database(s3_db_factory):
+def test_remote_database(s3_db_factory, mock_aws_env):
     keys = ('some', 'keys')
     dbpath = s3_db_factory(keys)
 
@@ -89,7 +47,7 @@ def test_invalid_url():
 
 
 @moto.mock_s3
-def test_nonexisting_url():
+def test_nonexisting_url(mock_aws_env):
     from terracotta import exceptions, get_driver
     driver = get_driver('s3://foo/db.sqlite')
     with pytest.raises(exceptions.InvalidDatabaseError):
@@ -98,7 +56,7 @@ def test_nonexisting_url():
 
 
 @moto.mock_s3
-def test_remote_database_cache(s3_db_factory, raster_file, monkeypatch):
+def test_remote_database_cache(s3_db_factory, raster_file, mock_aws_env):
     keys = ('some', 'keys')
     dbpath = s3_db_factory(keys)
 
@@ -134,26 +92,26 @@ def test_remote_database_cache(s3_db_factory, raster_file, monkeypatch):
 
 
 @moto.mock_s3
-def test_immutability(s3_db_factory, raster_file):
+def test_immutability(s3_db_factory, raster_file, mock_aws_env):
     keys = ('some', 'keys')
     dbpath = s3_db_factory(keys, datasets={('some', 'value'): str(raster_file)})
 
     from terracotta import get_driver
 
     driver = get_driver(dbpath)
 
-    with pytest.raises(NotImplementedError):
+    with pytest.raises(exceptions.DatabaseNotWritableError):
         driver.create(keys)
 
-    with pytest.raises(NotImplementedError):
+    with pytest.raises(exceptions.DatabaseNotWritableError):
         driver.insert(('some', 'value'), str(raster_file))
 
-    with pytest.raises(NotImplementedError):
+    with pytest.raises(exceptions.DatabaseNotWritableError):
         driver.delete(('some', 'value'))
 
 
 @moto.mock_s3
-def test_destructor(s3_db_factory, raster_file, capsys):
+def test_destructor(s3_db_factory, raster_file, capsys, mock_aws_env):
     keys = ('some', 'keys')
     dbpath = s3_db_factory(keys, datasets={('some', 'value'): str(raster_file)})