Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'v3' into pre-commit-ci-update-config
Browse files Browse the repository at this point in the history
dstansby authored Sep 8, 2024
2 parents 665f4a6 + 8e29f37 commit 052ee1f
Showing 62 changed files with 1,688 additions and 308 deletions.
66 changes: 66 additions & 0 deletions .github/workflows/gpu_test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: GPU Test V3

on:
push:
branches: [ v3 ]
pull_request:
branches: [ v3 ]
workflow_dispatch:

env:
LD_LIBRARY_PATH: /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
test:
name: py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }}

runs-on: gpu-runner
strategy:
matrix:
python-version: ['3.11']
numpy-version: ['2.0']
dependency-set: ["minimal"]

steps:
- uses: actions/checkout@v4
# - name: cuda-toolkit
# uses: Jimver/[email protected]
# id: cuda-toolkit
# with:
# cuda: '12.4.1'
- name: Set up CUDA
run: |
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb
sudo apt-get update
sudo apt-get -y install cuda-toolkit-12-6
echo "/usr/local/cuda/bin" >> $GITHUB_PATH
- name: GPU check
run: |
nvidia-smi
echo $PATH
echo $LD_LIBRARY_PATH
nvcc -V
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
- name: Install Hatch and CuPy
run: |
python -m pip install --upgrade pip
pip install hatch
- name: Set Up Hatch Env
run: |
hatch env create gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }}
hatch env run -e gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env
- name: Run Tests
run: |
hatch env run --env gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage
2 changes: 1 addition & 1 deletion .github/workflows/releases.yml
Original file line number Diff line number Diff line change
@@ -55,7 +55,7 @@ jobs:
with:
name: releases
path: dist
- uses: pypa/gh-action-pypi-publish@v1.9.0
- uses: pypa/gh-action-pypi-publish@v1.10.0
with:
user: __token__
password: ${{ secrets.pypi_password }}
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -51,6 +51,7 @@ coverage.xml

# Sphinx documentation
docs/_build/
docs/_autoapi

# PyBuilder
target/
14 changes: 12 additions & 2 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@ default_language_version:
python: python3
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: 'v0.6.2'
rev: v0.6.3
hooks:
- id: ruff
args: ["--fix", "--show-fixes"]
@@ -25,7 +25,7 @@ repos:
rev: v1.11.2
hooks:
- id: mypy
files: src
files: src|tests/v3/test_(api|array|buffer).py
additional_dependencies:
# Package dependencies
- asciitree
@@ -35,7 +35,17 @@ repos:
- numcodecs
- numpy
- typing_extensions
- universal-pathlib
# Tests
- pytest
# Zarr v2
- types-redis
- repo: https://github.com/scientific-python/cookie
rev: 2024.04.23
hooks:
- id: sp-repo-review
- repo: https://github.com/pre-commit/pygrep-hooks
rev: v1.10.0
hooks:
- id: rst-directive-colons
- id: rst-inline-touching-normal
1 change: 1 addition & 0 deletions docs/Makefile
Original file line number Diff line number Diff line change
@@ -52,6 +52,7 @@ help:
.PHONY: clean
clean:
rm -rf $(BUILDDIR)/*
rm -rf $(BUILDDIR)/../_autoapi

.PHONY: html
html:
2 changes: 1 addition & 1 deletion docs/api/index.rst
Original file line number Diff line number Diff line change
@@ -4,4 +4,4 @@ API Reference
.. toctree::
:maxdepth: 1

zarr
../_autoapi/zarr/index
5 changes: 0 additions & 5 deletions docs/api/zarr.rst

This file was deleted.

18 changes: 15 additions & 3 deletions docs/conf.py
Original file line number Diff line number Diff line change
@@ -57,9 +57,10 @@

autoapi_dirs = ['../src/zarr']
autoapi_add_toctree_entry = False
autoapi_generate_api_docs = False
autoapi_generate_api_docs = True
autoapi_member_order = "groupwise"
autoapi_root = "api"
autoapi_root = "_autoapi"
autoapi_keep_files = True


# Add any paths that contain templates here, relative to this directory.
@@ -172,8 +173,19 @@
html_logo = "_static/logo1.png"


def autoapi_skip_modules(app: sphinx.application.Sphinx, what: str, name: str, obj: object, skip: bool, options: dict[str, Any]) -> bool:
"""
Return True if a module should be skipped in th API docs.
"""
parts = name.split(".")
if what == "module" and (any(part.startswith("_") for part in parts) or "v2" in name or name.startswith("zarr.core")):
return True
return False


def setup(app: sphinx.application.Sphinx) -> None:
app.add_css_file("custom.css")
app.connect("autoapi-skip-member", autoapi_skip_modules)


# The name of an image file (relative to this directory) to use as a favicon of
@@ -339,7 +351,7 @@ def setup(app: sphinx.application.Sphinx) -> None:
# use in refs e.g:
# :ref:`comparison manual <python:comparisons>`
intersphinx_mapping = {
"python": ("https://docs.python.org/", None),
"python": ("https://docs.python.org/3/", None),
"numpy": ("https://numpy.org/doc/stable/", None),
"numcodecs": ("https://numcodecs.readthedocs.io/en/stable/", None),
}
48 changes: 45 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -74,10 +74,13 @@ jupyter = [
'ipytree>=0.2.2',
'ipywidgets>=8.0.0',
]
gpu = [
"cupy-cuda12x",
]
docs = [
'sphinx',
'sphinx<8',
'sphinx-autobuild>=2021.3.14',
'sphinx-autoapi',
'sphinx-autoapi<3.1',
'sphinx_design',
'sphinx-issues',
'sphinx-copybutton',
@@ -92,6 +95,7 @@ extra = [
]
optional = [
'lmdb',
'universal-pathlib',
]

[project.urls]
@@ -120,7 +124,7 @@ build.hooks.vcs.version-file = "src/zarr/_version.py"
[tool.hatch.envs.test]
dependencies = [
"numpy~={matrix:numpy}",
"universal_pathlib"
"universal_pathlib",
]
features = ["test", "extra"]

@@ -134,8 +138,34 @@ python = ["3.10", "3.11", "3.12"]
numpy = ["1.24", "1.26", "2.0"]
features = ["optional"]

[[tool.hatch.envs.test.matrix]]
python = ["3.10", "3.11", "3.12"]
numpy = ["1.24", "1.26", "2.0"]
features = ["gpu"]

[tool.hatch.envs.test.scripts]
run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov=tests"
run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov=tests"
run = "run-coverage --no-cov"
run-verbose = "run-coverage --verbose"
run-mypy = "mypy src"
run-hypothesis = "pytest --hypothesis-profile ci tests/v3/test_properties.py tests/v3/test_store/test_stateful*"
list-env = "pip list"

[tool.hatch.envs.gputest]
dependencies = [
"numpy~={matrix:numpy}",
"universal_pathlib",
]
features = ["test", "extra", "gpu"]

[[tool.hatch.envs.gputest.matrix]]
python = ["3.10", "3.11", "3.12"]
numpy = ["1.24", "1.26", "2.0"]
version = ["minimal"]

[tool.hatch.envs.gputest.scripts]
run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov=tests"
run = "run-coverage --no-cov"
run-verbose = "run-coverage --verbose"
run-mypy = "mypy src"
@@ -168,6 +198,7 @@ extend-exclude = [
"buck-out",
"build",
"dist",
"notebooks", # temporary, until we achieve compatibility with ruff ≥ 0.6
"venv",
"docs",
"src/zarr/v2/",
@@ -182,6 +213,7 @@ extend-select = [
"UP", # pyupgrade
"RSE",
"RUF",
"TCH", # flake8-type-checking
"TRY", # tryceratops
]
ignore = [
@@ -223,4 +255,14 @@ filterwarnings = [
"error:::zarr.*",
"ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning",
"ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning",
"ignore:Creating a zarr.buffer.gpu.*:UserWarning",
]
markers = [
"gpu: mark a test as requiring CuPy and GPU"
]

[tool.repo-review]
ignore = [
"PC111", # fix Python code in documentation - enable later
"PC180", # for JavaScript - not interested
]
10 changes: 5 additions & 5 deletions src/zarr/abc/codec.py
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
from __future__ import annotations

from abc import abstractmethod
from collections.abc import Awaitable, Callable, Iterable
from typing import TYPE_CHECKING, Any, Generic, TypeVar

import numpy as np

from zarr.abc.metadata import Metadata
from zarr.abc.store import ByteGetter, ByteSetter
from zarr.core.buffer import Buffer, NDBuffer
from zarr.core.chunk_grids import ChunkGrid
from zarr.core.common import ChunkCoords, concurrent_map
from zarr.core.config import config

if TYPE_CHECKING:
from collections.abc import Awaitable, Callable, Iterable

import numpy as np
from typing_extensions import Self

from zarr.abc.store import ByteGetter, ByteSetter
from zarr.core.array_spec import ArraySpec
from zarr.core.chunk_grids import ChunkGrid
from zarr.core.common import JSON
from zarr.core.indexing import SelectorTuple

4 changes: 2 additions & 2 deletions src/zarr/abc/metadata.py
Original file line number Diff line number Diff line change
@@ -6,9 +6,9 @@
if TYPE_CHECKING:
from typing_extensions import Self

from dataclasses import dataclass, fields
from zarr.core.common import JSON

from zarr.core.common import JSON
from dataclasses import dataclass, fields

__all__ = ["Metadata"]

13 changes: 8 additions & 5 deletions src/zarr/api/asynchronous.py
Original file line number Diff line number Diff line change
@@ -2,16 +2,12 @@

import asyncio
import warnings
from collections.abc import Iterable
from typing import Any, Literal, Union, cast
from typing import TYPE_CHECKING, Any, Literal, Union, cast

import numpy as np
import numpy.typing as npt

from zarr.abc.codec import Codec
from zarr.core.array import Array, AsyncArray
from zarr.core.buffer import NDArrayLike
from zarr.core.chunk_key_encodings import ChunkKeyEncoding
from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, MemoryOrder, ZarrFormat
from zarr.core.group import AsyncGroup
from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata
@@ -20,6 +16,13 @@
make_store_path,
)

if TYPE_CHECKING:
from collections.abc import Iterable

from zarr.abc.codec import Codec
from zarr.core.buffer import NDArrayLike
from zarr.core.chunk_key_encodings import ChunkKeyEncoding

__all__ = [
"consolidate_metadata",
"copy",
10 changes: 6 additions & 4 deletions src/zarr/api/synchronous.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
from __future__ import annotations

from typing import Any
from typing import TYPE_CHECKING, Any

import zarr.api.asynchronous as async_api
from zarr.core.array import Array, AsyncArray
from zarr.core.buffer import NDArrayLike
from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, ZarrFormat
from zarr.core.group import Group
from zarr.core.sync import sync
from zarr.store import StoreLike

if TYPE_CHECKING:
from zarr.core.buffer import NDArrayLike
from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, ZarrFormat
from zarr.store import StoreLike

__all__ = [
"consolidate_metadata",
5 changes: 4 additions & 1 deletion src/zarr/codecs/_v2.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
from __future__ import annotations

from dataclasses import dataclass
from typing import TYPE_CHECKING

import numcodecs
from numcodecs.compat import ensure_bytes, ensure_ndarray

from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec
from zarr.core.array_spec import ArraySpec
from zarr.core.buffer import Buffer, NDBuffer, default_buffer_prototype
from zarr.core.common import JSON, to_thread
from zarr.registry import get_ndbuffer_class

if TYPE_CHECKING:
from zarr.core.array_spec import ArraySpec


@dataclass(frozen=True)
class V2Compressor(ArrayBytesCodec):
Loading

0 comments on commit 052ee1f

Please sign in to comment.