Merge pull request #23 from d-v-b/mdurant_v3_fsspec

Mdurant v3 fsspec

pyproject.toml

@@ -31,7 +31,8 @@ dependencies = [
     'crc32c',
     'zstandard',
     'typing_extensions',
-    'donfig'
+    'donfig',
+    'pytest'
 ]
 dynamic = [
   "version",
@@ -112,8 +113,11 @@ extra-dependencies = [
     "pytest-cov",
     "msgpack",
     "lmdb",
+    "s3fs",
     "pytest-asyncio",
-    "moto",
+    "moto[s3]",
+    "flask-cors",
+    "flask",
     "requests",
     "mypy"
 ]

src/zarr/store/remote.py

@@ -6,7 +6,7 @@
 import fsspec
 
 from zarr.abc.store import Store
-from zarr.buffer import Buffer, BufferPrototype
+from zarr.buffer import Buffer, BufferPrototype, default_buffer_prototype
 from zarr.common import OpenMode
 from zarr.store.core import _dereference_path
 
@@ -79,7 +79,7 @@ def __repr__(self) -> str:
     async def get(
         self,
         key: str,
-        prototype: BufferPrototype,
+        prototype: BufferPrototype = default_buffer_prototype,
         byte_range: tuple[int | None, int | None] | None = None,
     ) -> Buffer | None:
         path = _dereference_path(self.path, key)

tests/v3/test_store.py → tests/v3/test_store/test_local.py

@@ -1,48 +1,12 @@
 from __future__ import annotations
 
-from typing import Any
-
 import pytest
 
 from zarr.buffer import Buffer
 from zarr.store.local import LocalStore
-from zarr.store.memory import MemoryStore
 from zarr.testing.store import StoreTests
 
 
-class TestMemoryStore(StoreTests[MemoryStore]):
-    store_cls = MemoryStore
-
-    def set(self, store: MemoryStore, key: str, value: Buffer) -> None:
-        store._store_dict[key] = value
-
-    def get(self, store: MemoryStore, key: str) -> Buffer:
-        return store._store_dict[key]
-
-    @pytest.fixture(scope="function", params=[None, {}])
-    def store_kwargs(self, request) -> dict[str, Any]:
-        return {"store_dict": request.param, "mode": "w"}
-
-    @pytest.fixture(scope="function")
-    def store(self, store_kwargs: dict[str, Any]) -> MemoryStore:
-        return self.store_cls(**store_kwargs)
-
-    def test_store_repr(self, store: MemoryStore) -> None:
-        assert str(store) == f"memory://{id(store._store_dict)}"
-
-    def test_store_supports_writes(self, store: MemoryStore) -> None:
-        assert store.supports_writes
-
-    def test_store_supports_listing(self, store: MemoryStore) -> None:
-        assert store.supports_listing
-
-    def test_store_supports_partial_writes(self, store: MemoryStore) -> None:
-        assert store.supports_partial_writes
-
-    def test_list_prefix(self, store: MemoryStore) -> None:
-        assert True
-
-
 class TestLocalStore(StoreTests[LocalStore]):
     store_cls = LocalStore
 

tests/v3/test_store/test_memory.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import pytest
+
+from zarr.buffer import Buffer
+from zarr.store.memory import MemoryStore
+from zarr.testing.store import StoreTests
+
+
+class TestMemoryStore(StoreTests[MemoryStore]):
+    store_cls = MemoryStore
+
+    def set(self, store: MemoryStore, key: str, value: Buffer) -> None:
+        store._store_dict[key] = value
+
+    def get(self, store: MemoryStore, key: str) -> Buffer:
+        return store._store_dict[key]
+
+    @pytest.fixture(scope="function", params=[None, {}])
+    def store_kwargs(self, request) -> dict[str, str | None | dict[str, Buffer]]:
+        return {"store_dict": request.param, "mode": "w"}
+
+    @pytest.fixture(scope="function")
+    def store(self, store_kwargs: str | None | dict[str, Buffer]) -> MemoryStore:
+        return self.store_cls(**store_kwargs)
+
+    def test_store_repr(self, store: MemoryStore) -> None:
+        assert str(store) == f"memory://{id(store._store_dict)}"
+
+    def test_store_supports_writes(self, store: MemoryStore) -> None:
+        assert store.supports_writes
+
+    def test_store_supports_listing(self, store: MemoryStore) -> None:
+        assert store.supports_listing
+
+    def test_store_supports_partial_writes(self, store: MemoryStore) -> None:
+        assert store.supports_partial_writes
+
+    def test_list_prefix(self, store: MemoryStore) -> None:
+        assert True

tests/v3/test_remote_store.py → tests/v3/test_store/test_remote.py

@@ -2,8 +2,10 @@
 
 import pytest
 
-from zarr.buffer import Buffer
+from zarr.buffer import Buffer, default_buffer_prototype
 from zarr.store import RemoteStore
+from zarr.sync import sync
+from zarr.testing.store import StoreTests
 
 s3fs = pytest.importorskip("s3fs")
 requests = pytest.importorskip("requests")
@@ -41,7 +43,7 @@ def get_boto3_client():
     return session.create_client("s3", endpoint_url=endpoint_uri)
 
 
-@pytest.fixture()
+@pytest.fixture(autouse=True, scope="session")
 def s3(s3_base):
     client = get_boto3_client()
     client.create_bucket(Bucket=test_bucket_name, ACL="public-read")
@@ -62,13 +64,51 @@ async def alist(it):
     return out
 
 
-async def test_basic(s3):
+async def test_basic():
     store = RemoteStore(f"s3://{test_bucket_name}", mode="w", endpoint_url=endpoint_uri, anon=False)
     assert not await alist(store.list())
     assert not await store.exists("foo")
     data = b"hello"
     await store.set("foo", Buffer.from_bytes(data))
     assert await store.exists("foo")
     assert (await store.get("foo")).to_bytes() == data
-    out = await store.get_partial_values([("foo", (1, None))])
+    out = await store.get_partial_values(
+        prototype=default_buffer_prototype, key_ranges=[("foo", (1, None))]
+    )
     assert out[0].to_bytes() == data[1:]
+
+
+class TestRemoteStoreS3(StoreTests[RemoteStore]):
+    store_cls = RemoteStore
+
+    @pytest.fixture(scope="function")
+    def store_kwargs(self) -> dict[str, str | bool]:
+        return {
+            "mode": "w",
+            "endpoint_url": endpoint_uri,
+            "anon": False,
+            "url": f"s3://{test_bucket_name}",
+        }
+
+    @pytest.fixture(scope="function")
+    def store(self, store_kwargs: dict[str, str | bool]) -> RemoteStore:
+        return self.store_cls(**store_kwargs)
+
+    def get(self, store: RemoteStore, key: str) -> Buffer:
+        return Buffer.from_bytes(sync(store._fs.cat(f"{store.path}/{key}")))
+
+    def set(self, store: RemoteStore, key: str, value: Buffer) -> None:
+        store._fs.write_bytes(f"{store.path}/{key}", value.to_bytes())
+
+    def test_store_repr(self, store: RemoteStore) -> None:
+        assert str(store) == f"Remote fsspec store: {store.path}"
+
+    def test_store_supports_writes(self, store: RemoteStore) -> None:
+        assert True
+
+    @pytest.mark.xfail
+    def test_store_supports_partial_writes(self, store: RemoteStore) -> None:
+        raise AssertionError
+
+    def test_store_supports_listing(self, store: RemoteStore) -> None:
+        assert True