diff --git a/.github/workflows/deploy-docs.yaml b/.github/workflows/deploy-docs.yaml index 1d8563ec..de9ce709 100644 --- a/.github/workflows/deploy-docs.yaml +++ b/.github/workflows/deploy-docs.yaml @@ -19,7 +19,8 @@ jobs: git config user.name github-actions[bot] git config user.email 41898282+github-actions[bot]@users.noreply.github.com - - uses: actions/setup-python@v5 + - id: setup-python + uses: actions/setup-python@v5 with: python-version: 3.x diff --git a/.github/workflows/publish-rust-library.yml b/.github/workflows/publish-rust-library.yml index 720da2ec..c9c72fdc 100644 --- a/.github/workflows/publish-rust-library.yml +++ b/.github/workflows/publish-rust-library.yml @@ -24,7 +24,7 @@ jobs: docker compose up -d minio - name: Wait for MinIO to be ready run: | - for i in {1..10}; do + for _ in {1..10}; do if curl --silent --fail http://minio:9000/minio/health/live; then break fi @@ -51,7 +51,7 @@ jobs: run: cargo install --locked cargo-deny - name: Check - if: matrix.os == 'ubuntu-latest' || github.event_name == 'push' + if: github.event_name == 'push' run: | just pre-commit diff --git a/.github/workflows/python-check.yaml b/.github/workflows/python-check.yaml index be56811b..7af17d3e 100644 --- a/.github/workflows/python-check.yaml +++ b/.github/workflows/python-check.yaml @@ -35,7 +35,7 @@ jobs: - name: Wait for MinIO to be ready run: | - for i in {1..10}; do + for _ in {1..10}; do if curl --silent --fail http://minio:9000/minio/health/live; then break fi @@ -50,10 +50,10 @@ jobs: uses: PyO3/maturin-action@v1 with: working-directory: icechunk-python - target: ${{ matrix.platform.target }} + # target: ${{ matrix.platform.target }} args: --release --out dist --find-interpreter sccache: 'true' - manylinux: ${{ matrix.platform.manylinux }} # https://github.com/PyO3/maturin-action/issues/245 + # manylinux: ${{ matrix.platform.manylinux }} # https://github.com/PyO3/maturin-action/issues/245 - name: Cargo lint + format shell: bash @@ -81,6 +81,15 @@ jobs: pip install icechunk['test'] --find-links dist --force-reinstall ruff check + - name: Restore cached hypothesis directory + id: restore-hypothesis-cache + uses: actions/cache/restore@v4 + with: + path: icechunk-python/.hypothesis/ + key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} + restore-keys: | + cache-hypothesis- + - name: pytest shell: bash working-directory: icechunk-python @@ -90,3 +99,12 @@ jobs: source .venv/bin/activate pip install icechunk['test'] --find-links dist --force-reinstall pytest + + # explicitly save the cache so it gets updated, also do this even if it fails. + - name: Save cached hypothesis directory + id: save-hypothesis-cache + if: always() + uses: actions/cache/save@v4 + with: + path: icechunk-python/.hypothesis/ + key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} diff --git a/.github/workflows/python-ci.yaml b/.github/workflows/python-ci.yaml index 7ad19760..97f38b15 100644 --- a/.github/workflows/python-ci.yaml +++ b/.github/workflows/python-ci.yaml @@ -46,7 +46,7 @@ jobs: docker compose up -d minio - name: Wait for MinIO to be ready run: | - for i in {1..10}; do + for _ in {1..10}; do if curl --silent --fail http://minio:9000/minio/health/live; then break fi diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index 34fd7075..abb81bf0 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -57,7 +57,7 @@ jobs: - name: Wait for MinIO to be ready run: | - for i in {1..10}; do + for _ in {1..10}; do if curl --silent --fail http://minio:9000/minio/health/live; then break fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ad90a36..f2d8cad8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,8 @@ repos: hooks: - id: trailing-whitespace - id: end-of-file-fixer - # - id: check-yaml + - id: check-yaml + exclude: docs/mkdocs.yml - id: debug-statements - id: mixed-line-ending - repo: https://github.com/astral-sh/ruff-pre-commit @@ -14,6 +15,17 @@ repos: - id: ruff-format - id: ruff args: ["--fix", "--show-fixes", "icechunk-python/"] + - repo: https://github.com/rhysd/actionlint + rev: v1.6.26 + hooks: + - id: actionlint + files: ".github/workflows/" + args: ["-ignore", "SC1090", "-ignore", "SC2046", "-ignore", "SC2086", "-ignore", "SC2129", "-ignore", "SC2155"] + - repo: https://github.com/codespell-project/codespell + # Configuration for codespell is in .codespellrc + rev: v2.3.0 + hooks: + - id: codespell - repo: local hooks: - id: format-lint @@ -22,10 +34,5 @@ repos: entry: just pre-commit language: system pass_filenames: false - - repo: https://github.com/codespell-project/codespell - # Configuration for codespell is in .codespellrc - rev: v2.3.0 - hooks: - - id: codespell exclude: 'tests/data/.*' diff --git a/icechunk-python/pyproject.toml b/icechunk-python/pyproject.toml index e0a3e825..40192505 100644 --- a/icechunk-python/pyproject.toml +++ b/icechunk-python/pyproject.toml @@ -37,6 +37,7 @@ test = [ "ruff", "dask", "distributed", + "hypothesis", ] [tool.maturin] diff --git a/icechunk-python/tests/test_zarr/test_properties.py b/icechunk-python/tests/test_zarr/test_properties.py new file mode 100644 index 00000000..73381784 --- /dev/null +++ b/icechunk-python/tests/test_zarr/test_properties.py @@ -0,0 +1,51 @@ +import numpy as np +import pytest +from numpy.testing import assert_array_equal + +from icechunk import IcechunkStore, StorageConfig + +pytest.importorskip("hypothesis") + +import hypothesis.strategies as st +from hypothesis import assume, given, settings + +from zarr.testing.strategies import arrays, numpy_arrays + +readable_characters = st.characters( + # only use characters within the "Latin Extended-A" subset of unicode + categories=["L", "N", "P"], + max_codepoint=0x017F, +) +_attr_keys = st.text(readable_characters, min_size=1) +_attr_values = st.recursive( + st.none() | st.booleans() | st.text(readable_characters, min_size=1, max_size=5), + lambda children: st.lists(children) | st.dictionaries(_attr_keys, children), + max_leaves=3, +) +simple_attrs = st.none() + +icechunk_stores = st.builds( + IcechunkStore.create, + storage=st.builds(StorageConfig.memory, prefix=st.just("prefix")), + read_only=st.just(False), +) + + +@settings(report_multiple_bugs=True, deadline=None) +@given(data=st.data(), nparray=numpy_arrays(zarr_formats=st.just(3))) +def test_roundtrip(data: st.DataObject, nparray) -> None: + # TODO: support size-0 arrays GH392 + assume(nparray.size > 0) + # TODO: fix complex fill values GH391 + assume(not np.iscomplexobj(nparray)) + + zarray = data.draw( + arrays( + stores=icechunk_stores, + arrays=st.just(nparray), + zarr_formats=st.just(3), + # TODO: use all attrs once fixed GH393 + attrs=simple_attrs, + ) + ) + assert_array_equal(nparray, zarray[:])