Skip to content

Commit

Permalink
Merge branch 'main' into conv-cal-random
Browse files Browse the repository at this point in the history
  • Loading branch information
aulemahal authored Apr 15, 2024
2 parents b531f0e + b004af5 commit b581e1f
Show file tree
Hide file tree
Showing 197 changed files with 18,123 additions and 4,658 deletions.
14 changes: 7 additions & 7 deletions .github/workflows/ci-additional.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,13 @@ jobs:
runs-on: "ubuntu-latest"
needs: detect-ci-trigger
if: needs.detect-ci-trigger.outputs.triggered == 'false'

defaults:
run:
shell: bash -l {0}

env:
CONDA_ENV_FILE: ci/requirements/environment.yml
PYTHON_VERSION: "3.11"

steps:
- uses: actions/checkout@v4
with:
Expand Down Expand Up @@ -79,7 +78,8 @@ jobs:
#
# If dependencies emit warnings we can't do anything about, add ignores to
# `xarray/tests/__init__.py`.
python -m pytest --doctest-modules xarray --ignore xarray/tests -Werror
# [MHS, 01/25/2024] Skip datatree_ documentation remove after #8572
python -m pytest --doctest-modules xarray --ignore xarray/tests --ignore xarray/datatree_ -Werror
mypy:
name: Mypy
Expand Down Expand Up @@ -127,7 +127,7 @@ jobs:
python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report xarray/
- name: Upload mypy coverage to Codecov
uses: codecov/codecov-action@v3.1.4
uses: codecov/codecov-action@v4.2.0
with:
file: mypy_report/cobertura.xml
flags: mypy
Expand Down Expand Up @@ -181,7 +181,7 @@ jobs:
python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report xarray/
- name: Upload mypy coverage to Codecov
uses: codecov/codecov-action@v3.1.4
uses: codecov/codecov-action@v4.2.0
with:
file: mypy_report/cobertura.xml
flags: mypy39
Expand Down Expand Up @@ -242,7 +242,7 @@ jobs:
python -m pyright xarray/
- name: Upload pyright coverage to Codecov
uses: codecov/codecov-action@v3.1.4
uses: codecov/codecov-action@v4.2.0
with:
file: pyright_report/cobertura.xml
flags: pyright
Expand Down Expand Up @@ -301,7 +301,7 @@ jobs:
python -m pyright xarray/
- name: Upload pyright coverage to Codecov
uses: codecov/codecov-action@v3.1.4
uses: codecov/codecov-action@v4.2.0
with:
file: pyright_report/cobertura.xml
flags: pyright39
Expand Down
10 changes: 9 additions & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,14 @@ jobs:
run: |
python -c "import xarray"
- name: Restore cached hypothesis directory
uses: actions/cache@v4
with:
path: .hypothesis/
key: cache-hypothesis
enableCrossOsArchive: true
save-always: true

- name: Run tests
run: python -m pytest -n 4
--timeout 180
Expand All @@ -143,7 +151,7 @@ jobs:
path: pytest.xml

- name: Upload code coverage to Codecov
uses: codecov/codecov-action@v3.1.4
uses: codecov/codecov-action@v4.2.0
with:
file: ./coverage.xml
flags: unittests
Expand Down
116 changes: 116 additions & 0 deletions .github/workflows/hypothesis.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
name: Slow Hypothesis CI
on:
push:
branches:
- "main"
pull_request:
branches:
- "main"
types: [opened, reopened, synchronize, labeled]
schedule:
- cron: "0 0 * * *" # Daily “At 00:00” UTC
workflow_dispatch: # allows you to trigger manually

jobs:
detect-ci-trigger:
name: detect ci trigger
runs-on: ubuntu-latest
if: |
github.repository == 'pydata/xarray'
&& (github.event_name == 'push' || github.event_name == 'pull_request')
outputs:
triggered: ${{ steps.detect-trigger.outputs.trigger-found }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- uses: xarray-contrib/ci-trigger@v1
id: detect-trigger
with:
keyword: "[skip-ci]"

hypothesis:
name: Slow Hypothesis Tests
runs-on: "ubuntu-latest"
needs: detect-ci-trigger
if: |
always()
&& (
(github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
|| needs.detect-ci-trigger.outputs.triggered == 'true'
|| contains( github.event.pull_request.labels.*.name, 'run-slow-hypothesis')
)
defaults:
run:
shell: bash -l {0}

env:
CONDA_ENV_FILE: ci/requirements/environment.yml
PYTHON_VERSION: "3.12"

steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # Fetch all history for all branches and tags.

- name: set environment variables
run: |
echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
- name: Setup micromamba
uses: mamba-org/setup-micromamba@v1
with:
environment-file: ci/requirements/environment.yml
environment-name: xarray-tests
create-args: >-
python=${{env.PYTHON_VERSION}}
pytest-reportlog
cache-environment: true
cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"

- name: Install xarray
run: |
python -m pip install --no-deps -e .
- name: Version info
run: |
conda info -a
conda list
python xarray/util/print_versions.py
# https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
- name: Restore cached hypothesis directory
id: restore-hypothesis-cache
uses: actions/cache/restore@v4
with:
path: .hypothesis/
key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
restore-keys: |
cache-hypothesis-
- name: Run slow Hypothesis tests
if: success()
id: status
run: |
python -m pytest --hypothesis-show-statistics --run-slow-hypothesis properties/*.py \
--report-log output-${{ matrix.python-version }}-log.jsonl
# explicitly save the cache so it gets updated, also do this even if it fails.
- name: Save cached hypothesis directory
id: save-hypothesis-cache
if: always() && steps.status.outcome != 'skipped'
uses: actions/cache/save@v4
with:
path: .hypothesis/
key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}

- name: Generate and publish the report
if: |
failure()
&& steps.status.outcome == 'failure'
&& github.event_name == 'schedule'
&& github.repository_owner == 'pydata'
uses: xarray-contrib/issue-from-pytest-log@v1
with:
log-path: output-${{ matrix.python-version }}-log.jsonl
issue-title: "Nightly Hypothesis tests failed"
issue-label: "topic-hypothesis"
2 changes: 1 addition & 1 deletion .github/workflows/nightly-wheels.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
fi
- name: Upload wheel
uses: scientific-python/upload-nightly-action@main
uses: scientific-python/upload-nightly-action@b67d7fcc0396e1128a474d1ab2b48aa94680f9fc # 0.5.0
with:
anaconda_nightly_upload_token: ${{ secrets.ANACONDA_NIGHTLY }}
artifacts_path: dist
4 changes: 2 additions & 2 deletions .github/workflows/pypi-release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ jobs:
path: dist
- name: Publish package to TestPyPI
if: github.event_name == 'push'
uses: pypa/[email protected].11
uses: pypa/[email protected].14
with:
repository_url: https://test.pypi.org/legacy/
verbose: true
Expand All @@ -111,6 +111,6 @@ jobs:
name: releases
path: dist
- name: Publish package to PyPI
uses: pypa/[email protected].11
uses: pypa/[email protected].14
with:
verbose: true
4 changes: 2 additions & 2 deletions .github/workflows/upstream-dev-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.11"]
python-version: ["3.12"]
steps:
- uses: actions/checkout@v4
with:
Expand Down Expand Up @@ -143,7 +143,7 @@ jobs:
run: |
python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report
- name: Upload mypy coverage to Codecov
uses: codecov/codecov-action@v3.1.4
uses: codecov/codecov-action@v4.2.0
with:
file: mypy_report/cobertura.xml
flags: mypy
Expand Down
15 changes: 5 additions & 10 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# https://pre-commit.com/
ci:
autoupdate_schedule: monthly
exclude: 'xarray/datatree_.*'
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
Expand All @@ -10,32 +11,26 @@ repos:
- id: check-yaml
- id: debug-statements
- id: mixed-line-ending
- repo: https://github.com/MarcoGorelli/absolufy-imports
rev: v0.3.1
hooks:
- id: absolufy-imports
name: absolufy-imports
files: ^xarray/
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: 'v0.1.9'
rev: 'v0.3.4'
hooks:
- id: ruff
args: ["--fix", "--show-fixes"]
# https://github.com/python/black#version-control-integration
- repo: https://github.com/psf/black-pre-commit-mirror
rev: 23.12.1
rev: 24.3.0
hooks:
- id: black-jupyter
- repo: https://github.com/keewis/blackdoc
rev: v0.3.9
hooks:
- id: blackdoc
exclude: "generate_aggregations.py"
additional_dependencies: ["black==23.12.1"]
additional_dependencies: ["black==24.3.0"]
- id: blackdoc-autoupdate-black
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.8.0
rev: v1.9.0
hooks:
- id: mypy
# Copied from setup.cfg
Expand Down
2 changes: 1 addition & 1 deletion HOW_TO_RELEASE.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ upstream https://github.com/pydata/xarray (push)
```sh
pytest
```
8. Check that the ReadTheDocs build is passing on the `main` branch.
8. Check that the [ReadTheDocs build](https://readthedocs.org/projects/xray/) is passing on the `latest` build version (which is built from the `main` branch).
9. Issue the release on GitHub. Click on "Draft a new release" at
<https://github.com/pydata/xarray/releases>. Type in the version number (with a "v")
and paste the release summary in the notes.
Expand Down
1 change: 1 addition & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
prune xarray/datatree_*
22 changes: 17 additions & 5 deletions ci/install-upstream-wheels.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@
# install cython for building cftime without build isolation
micromamba install "cython>=0.29.20" py-cpuinfo
# temporarily (?) remove numbagg and numba
micromamba remove -y numba numbagg
micromamba remove -y numba numbagg sparse
# temporarily remove numexpr
micromamba remove -y numexpr
# temporarily remove backends
micromamba remove -y cf_units h5py hdf5 netcdf4
micromamba remove -y cf_units hdf5 h5py netcdf4
# forcibly remove packages to avoid artifacts
conda uninstall -y --force \
micromamba remove -y --force \
numpy \
scipy \
pandas \
Expand All @@ -30,8 +32,17 @@ python -m pip install \
scipy \
matplotlib \
pandas
# for some reason pandas depends on pyarrow already.
# Remove once a `pyarrow` version compiled with `numpy>=2.0` is on `conda-forge`
python -m pip install \
-i https://pypi.fury.io/arrow-nightlies/ \
--prefer-binary \
--no-deps \
--pre \
--upgrade \
pyarrow
# without build isolation for packages compiling against numpy
# TODO: remove once there are `numpy>=2.0` builds for numcodecs and cftime
# TODO: remove once there are `numpy>=2.0` builds for these
python -m pip install \
--no-deps \
--upgrade \
Expand All @@ -51,13 +62,14 @@ python -m pip install \
--no-deps \
--upgrade \
git+https://github.com/dask/dask \
git+https://github.com/dask/dask-expr \
git+https://github.com/dask/distributed \
git+https://github.com/zarr-developers/zarr \
git+https://github.com/pypa/packaging \
git+https://github.com/hgrecco/pint \
git+https://github.com/pydata/sparse \
git+https://github.com/intake/filesystem_spec \
git+https://github.com/SciTools/nc-time-axis \
git+https://github.com/xarray-contrib/flox \
git+https://github.com/dgasmith/opt_einsum
# git+https://github.com/pydata/sparse
# git+https://github.com/h5netcdf/h5netcdf
1 change: 1 addition & 0 deletions ci/requirements/all-but-dask.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ channels:
dependencies:
- black
- aiobotocore
- array-api-strict
- boto3
- bottleneck
- cartopy
Expand Down
5 changes: 3 additions & 2 deletions ci/requirements/doc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ dependencies:
- cartopy
- cfgrib
- dask-core>=2022.1
- dask-expr
- hypothesis>=6.75.8
- h5netcdf>=0.13
- ipykernel
Expand Down Expand Up @@ -37,9 +38,9 @@ dependencies:
- sphinx-design
- sphinx-inline-tabs
- sphinx>=5.0
- sphinxext-opengraph
- sphinxext-rediraffe
- zarr>=2.10
- pip:
- sphinxext-rediraffe
- sphinxext-opengraph
# relative to this file. Needs to be editable to be accepted.
- -e ../..
Loading

0 comments on commit b581e1f

Please sign in to comment.