Commit stubs

.github/workflows/CI.yml

@@ -12,6 +12,7 @@ concurrency:
 
 env:
   CARGO_TERM_COLOR: always
+  PYTEST_ADDOPTS: '--color=yes'
 
 jobs:
   build_and_test:
@@ -49,10 +50,17 @@ jobs:
 
       - name: Install python deps + Build
         run: |
-          uv pip install --system -e ".[test]" maturin --verbose
+          uv pip install --system -e ".[test,dev]" --verbose
 
       - name: Python Tests
         run: pytest -n auto
 
       - name: Rust Tests
         run: cargo test
+
+      - name: Check formatting
+        # see “Type hints” section in contributing.md
+        run: |
+          cargo run --bin stub_gen
+          pre-commit run --all-files --show-diff-on-failure || true
+          git diff --exit-code HEAD

.gitignore

@@ -10,7 +10,6 @@ __pycache__/
 
 # Build
 *.so
-*.pyi
 /target/
 /dist/
 /docs/_build/

docs/contributing.md

@@ -32,10 +32,9 @@ for parallelized tests.  Most tests have been copied from the `zarr-python` repo
 
 ## Type hints
 
-When authoring Python code, your IDE will not be able to analyze the extension module `zarrs._internal`.
-But thanks to [`pyo3-stub-gen`][], we can generate type stubs for it!
+Thanks to [`pyo3-stub-gen`][], we can generate type stubs for the `zarrs._internal` module.
+If the “Check formatting” CI step fails, run `cargo run --bin stub_gen`, then `pre-commit run --all-files`, and commit the changes.
 
-To build the stub generator, run `cargo build --bin stub_gen`.
-Afterwards, whenever you `cargo build`, `maturin build` or interact with your editor’s rust language server (e.g. `rust-analyzer`), the type hints will be updated.
+Once `maturin` can be run as a `hatchling` plugin, this can be made automatic.
 
 [`pyo3-stub-gen`]: https://github.com/Jij-Inc/pyo3-stub-gen

pyproject.toml

@@ -44,21 +44,21 @@ test = [
     "hypothesis",
     "pytest-xdist",
 ]
-dev = ["maturin", "pip"]
+dev = ["maturin", "pip", "pre-commit"]
 doc = ["sphinx>=7.4.6", "myst-parser"]
 
 [tool.maturin]
 python-source = "python"
 module-name = "zarrs._internal"
 features = ["pyo3/extension-module"]
 
-
 [tool.pytest.ini_options]
 minversion = "7"
 testpaths = ["tests"]
 log_cli_level = "INFO"
 xfail_strict = true
 asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
 doctest_optionflags = [
     "NORMALIZE_WHITESPACE",
     "ELLIPSIS",
@@ -112,5 +112,7 @@ ignore = [
     # allow I, O, l as variable names -> I is the identity matrix, i, j, k, l is reasonable indexing notation
     "E741",
 ]
+[tool.ruff.lint.per-file-ignores]
+"**/*.pyi" = ["ICN001"]
 [tool.ruff.lint.isort]
 known-first-party = ["zarrs"]

python/zarrs/_internal.pyi

@@ -0,0 +1,47 @@
+# This file is automatically generated by pyo3_stub_gen
+# ruff: noqa: E501, F401
+
+import typing
+
+import numpy
+import numpy.typing
+
+class CodecPipelineImpl:
+    def __new__(
+        cls,
+        metadata,
+        *,
+        validate_checksums=...,
+        store_empty_chunks=...,
+        chunk_concurrent_minimum=...,
+        chunk_concurrent_maximum=...,
+        num_threads=...,
+    ): ...
+    def retrieve_chunks_and_apply_index(
+        self,
+        chunk_descriptions: typing.Sequence[
+            tuple[
+                tuple[str, typing.Sequence[int], str, typing.Sequence[int]],
+                typing.Sequence[slice],
+                typing.Sequence[slice],
+            ]
+        ],
+        value: numpy.NDArray[typing.Any],
+    ) -> None: ...
+    def retrieve_chunks(
+        self,
+        chunk_descriptions: typing.Sequence[
+            tuple[str, typing.Sequence[int], str, typing.Sequence[int]]
+        ],
+    ) -> list[numpy.typing.NDArray[numpy.uint8]]: ...
+    def store_chunks_with_indices(
+        self,
+        chunk_descriptions: typing.Sequence[
+            tuple[
+                tuple[str, typing.Sequence[int], str, typing.Sequence[int]],
+                typing.Sequence[slice],
+                typing.Sequence[slice],
+            ]
+        ],
+        value: numpy.NDArray[typing.Any],
+    ) -> None: ...

src/lib.rs

@@ -242,6 +242,7 @@ impl CodecPipelineImpl {
 impl CodecPipelineImpl {
     #[pyo3(signature = (
         metadata,
+        *,
         validate_checksums=None,
         store_empty_chunks=None,
         chunk_concurrent_minimum=None,

tests/test_codecs.py

@@ -305,7 +305,7 @@ def test_invalid_metadata(store: Store) -> None:
             ],
         )
     spath4 = StorePath(store, "invalid_missing_bytes_codec")
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=r".*[Cc]odec.*required"):
         Array.create(
             spath4,
             shape=(16, 16),
@@ -317,7 +317,9 @@ def test_invalid_metadata(store: Store) -> None:
             ],
         )
     spath5 = StorePath(store, "invalid_inner_chunk_shape")
-    with pytest.raises(ValueError):
+    with pytest.raises(
+        ValueError, match=r".*shard.*chunk_shape.*array.*shape.*need.*same.*dimensions"
+    ):
         Array.create(
             spath5,
             shape=(16, 16),
@@ -329,7 +331,9 @@ def test_invalid_metadata(store: Store) -> None:
             ],
         )
     spath6 = StorePath(store, "invalid_inner_chunk_shape")
-    with pytest.raises(ValueError):
+    with pytest.raises(
+        ValueError, match=r".*array.*chunk_shape.*divisible.*shard.*chunk_shape"
+    ):
         Array.create(
             spath6,
             shape=(16, 16),

tests/test_transpose.py

@@ -92,7 +92,7 @@ def test_transpose_invalid(
     data = np.arange(0, 256, dtype="uint16").reshape((1, 32, 8))
     spath = StorePath(store, "transpose_invalid")
     for order in [(1, 0), (3, 2, 1), (3, 3, 1)]:
-        with pytest.raises(ValueError):
+        with pytest.raises(ValueError, match=r".*order"):
             Array.create(
                 spath,
                 shape=data.shape,