From bf3e0e9c6794c116f67c0a2b4588ebd948092596 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 29 Jan 2024 12:07:37 +0100 Subject: [PATCH 1/6] ci: chain requirements to ensure consistency --- .github/workflows/docs.yml | 13 ++++++------- .github/workflows/doctests.yaml | 7 +++---- .github/workflows/flake8.yml | 1 - .github/workflows/mxnet_nightly.yml | 17 ++++++++--------- .github/workflows/pypi.yml | 1 - .../workflows/test_release_unix_nightly.yml | 19 +++++++++---------- .../workflows/test_release_win32_nightly.yml | 19 ++++++++----------- .github/workflows/tests-nixtla.yaml | 9 ++++----- .github/workflows/tests-nursery.yml | 19 ++++++++++--------- .github/workflows/tests-prophet.yaml | 6 +++--- .github/workflows/tests-r.yml | 7 +++---- .github/workflows/tests-torch.yml | 11 +++++------ .github/workflows/tests-xgboost.yaml | 8 ++++---- .github/workflows/tests.yml | 7 +++---- 14 files changed, 66 insertions(+), 78 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9ee6ed851a..668c07bd4c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -27,13 +27,12 @@ jobs: - name: Install dependencies run: | - python -m pip install -U pip - pip install ".[arrow]" - pip install -r requirements/requirements-docs.txt - pip install -r requirements/requirements-mxnet.txt - pip install -r requirements/requirements-pytorch.txt - pip install -r requirements/requirements-rotbaum.txt - pip install -r requirements/requirements-extras-statsforecast.txt + pip install ".[arrow]" \ + -r requirements/requirements-docs.txt \ + -r requirements/requirements-mxnet.txt \ + -r requirements/requirements-pytorch.txt \ + -r requirements/requirements-rotbaum.txt \ + -r requirements/requirements-extras-statsforecast.txt - name: Install Pandoc run: | diff --git a/.github/workflows/doctests.yaml b/.github/workflows/doctests.yaml index a1cf4258f0..d29af1cf29 100644 --- a/.github/workflows/doctests.yaml +++ b/.github/workflows/doctests.yaml @@ -21,10 +21,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install other dependencies run: | - python -m pip install -U pip - pip install -e . - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-arrow.txt + pip install -e . \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-arrow.txt - name: Test with pytest run: | pytest -n2 --doctest-modules \ diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index 3828854614..36cda1470e 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -14,7 +14,6 @@ jobs: python-version: '3.8' - name: Install dependencies run: | - python -m pip install -U pip pip install ruff pip install docformatter[tomli]==1.5.0 - name: Ruff (Flake8) diff --git a/.github/workflows/mxnet_nightly.yml b/.github/workflows/mxnet_nightly.yml index c30f3450af..b95e3d491c 100644 --- a/.github/workflows/mxnet_nightly.yml +++ b/.github/workflows/mxnet_nightly.yml @@ -26,15 +26,14 @@ jobs: run: | python -m pip install pip==20.2 pip install --pre "mxnet<2" -f https://dist.mxnet.io/python - pip install torch==1.10.0+cpu --no-cache-dir -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install pytorch-lightning~=1.5 - pip install -e ".[shell]" - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-sagemaker-sdk.txt - pip install -r requirements/requirements-extras-m-competitions.txt - pip install -r requirements/requirements-rotbaum.txt - pip install -r requirements/requirements-extras-anomaly-evaluation.txt - pip install -r requirements/requirements-extras-autogluon.txt + pip install "torch==1.10.0" "pytorch-lightning~=1.5" -f https://download.pytorch.org/whl/cpu/torch_stable.html + pip install -e ".[shell]" \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-sagemaker-sdk.txt \ + -r requirements/requirements-extras-m-competitions.txt \ + -r requirements/requirements-rotbaum.txt \ + -r requirements/requirements-extras-anomaly-evaluation.txt \ + -r requirements/requirements-extras-autogluon.txt - name: Test with pytest run: | pytest -m 'not (gpu or serial)' --cov src/gluonts --cov-report=term --cov-report xml test diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index d2ca008f29..cc756c6155 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -18,7 +18,6 @@ jobs: python-version: '3.8' - name: Install dependencies run: | - python -m pip install -U pip python -m pip install setuptools wheel - name: Create `dist` folder run: | diff --git a/.github/workflows/test_release_unix_nightly.yml b/.github/workflows/test_release_unix_nightly.yml index 3fa9307d44..84c6746788 100644 --- a/.github/workflows/test_release_unix_nightly.yml +++ b/.github/workflows/test_release_unix_nightly.yml @@ -24,16 +24,15 @@ jobs: git clone https://github.com/awslabs/gluonts --branch $(curl https://api.github.com/repos/awslabs/gluonts/releases/latest | grep tag_name | cut -d : -f 2,3 | tr -d \"\ | tr -d \,\ ) cd gluon-ts python -m pip install pip==20.2 - pip install mxnet~=1.8.0 - pip install torch==1.10.0+cpu --no-cache-dir -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install pytorch-lightning~=1.5 - pip install -e ".[shell]" - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-sagemaker-sdk.txt - pip install -r requirements/requirements-extras-m-competitions.txt - pip install -r requirements/requirements-rotbaum.txt - pip install -r requirements/requirements-extras-anomaly-evaluation.txt - pip install -r requirements/requirements-extras-autogluon.txt + pip install "mxnet~=1.8.0" "torch==1.10.0" "pytorch-lightning~=1.5" \ + -f https://download.pytorch.org/whl/cpu/torch_stable.html + pip install -e ".[shell]" \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-sagemaker-sdk.txt \ + -r requirements/requirements-extras-m-competitions.txt \ + -r requirements/requirements-rotbaum.txt \ + -r requirements/requirements-extras-anomaly-evaluation.txt \ + -r requirements/requirements-extras-autogluon.txt - name: Test with pytest run: | cd gluon-ts diff --git a/.github/workflows/test_release_win32_nightly.yml b/.github/workflows/test_release_win32_nightly.yml index 961e921287..35a416182a 100644 --- a/.github/workflows/test_release_win32_nightly.yml +++ b/.github/workflows/test_release_win32_nightly.yml @@ -25,17 +25,14 @@ jobs: $tmp=$tmp.psobject.properties.value.trim() git clone https://github.com/awslabs/gluonts --branch $tmp cd gluon-ts - python -m pip install -U pip - pip install mxnet~=1.7.0 - pip install torch - pip install pytorch-lightning~=1.5 - pip install -e ".[shell]" - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-sagemaker-sdk.txt - pip install -r requirements/requirements-extras-m-competitions.txt - pip install -r requirements/requirements-rotbaum.txt - pip install -r requirements/requirements-extras-anomaly-evaluation.txt - pip install -r requirements/requirements-extras-autogluon.txt + pip install "mxnet~=1.7.0" torch "pytorch-lightning~=1.5" + pip install -e ".[shell]" \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-sagemaker-sdk.txt \ + -r requirements/requirements-extras-m-competitions.txt \ + -r requirements/requirements-rotbaum.txt \ + -r requirements/requirements-extras-anomaly-evaluation.txt \ + -r requirements/requirements-extras-autogluon.txt shell: powershell - name: Test with pytest run: | diff --git a/.github/workflows/tests-nixtla.yaml b/.github/workflows/tests-nixtla.yaml index c9cca4cccf..86aefed138 100644 --- a/.github/workflows/tests-nixtla.yaml +++ b/.github/workflows/tests-nixtla.yaml @@ -21,11 +21,10 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install other dependencies run: | - python -m pip install -U pip - pip install . - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-statsforecast.txt - pip install -r requirements/requirements-extras-hierarchicalforecast.txt + pip install . \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-statsforecast.txt \ + -r requirements/requirements-extras-hierarchicalforecast.txt - name: Test with pytest run: | pytest -n2 --doctest-modules test/ext/statsforecast diff --git a/.github/workflows/tests-nursery.yml b/.github/workflows/tests-nursery.yml index b719f4e8fb..5bca25ec1c 100644 --- a/.github/workflows/tests-nursery.yml +++ b/.github/workflows/tests-nursery.yml @@ -29,15 +29,16 @@ jobs: run: pip install mxnet~=1.7.0 - name: Install other dependencies run: | - python -m pip install -U pip - pip install torch==1.10.0+cpu --no-cache-dir -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install pytorch-lightning~=1.5 - pip install -e ".[shell]" - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-sagemaker-sdk.txt - pip install -r requirements/requirements-extras-m-competitions.txt - pip install -r requirements/requirements-extras-anomaly-evaluation.txt - pip install -r requirements/requirements-extras-autogluon.txt + pip install "torch==1.10.0" "pytorch-lightning~=1.5" \ + -f https://download.pytorch.org/whl/cpu/torch_stable.html + pip install + pip install -e ".[shell]" \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-sagemaker-sdk.txt \ + -r requirements/requirements-extras-m-competitions.txt \ + -r requirements/requirements-rotbaum.txt \ + -r requirements/requirements-extras-anomaly-evaluation.txt \ + -r requirements/requirements-extras-autogluon.txt - name: Test with pytest run: | pytest -n2 --doctest-modules test/nursery diff --git a/.github/workflows/tests-prophet.yaml b/.github/workflows/tests-prophet.yaml index bd2f33b620..c2e7858908 100644 --- a/.github/workflows/tests-prophet.yaml +++ b/.github/workflows/tests-prophet.yaml @@ -22,9 +22,9 @@ jobs: - name: Install other dependencies run: | python -m pip install -U pip - pip install . - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-prophet.txt + pip install . \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-prophet.txt - name: Test with pytest run: | pytest -n2 --doctest-modules test/ext/prophet diff --git a/.github/workflows/tests-r.yml b/.github/workflows/tests-r.yml index c7e3ccac9d..52c8ab6206 100644 --- a/.github/workflows/tests-r.yml +++ b/.github/workflows/tests-r.yml @@ -38,10 +38,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Python dependencies run: | - python -m pip install -U pip - pip install . - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-r.txt + pip install . \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-r.txt - name: Test with pytest run: | pytest -n2 test/ext/r_forecast diff --git a/.github/workflows/tests-torch.yml b/.github/workflows/tests-torch.yml index 08cff85e5f..3763f6f01f 100644 --- a/.github/workflows/tests-torch.yml +++ b/.github/workflows/tests-torch.yml @@ -21,12 +21,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install -U pip - pip install ".[arrow]" - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-m-competitions.txt - pip install -r requirements/requirements-pytorch.txt - pip install -r requirements/requirements-extras-cpflows.txt + pip install ".[arrow]" \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-m-competitions.txt \ + -r requirements/requirements-pytorch.txt \ + -r requirements/requirements-extras-cpflows.txt - name: Test with pytest run: | pytest -n2 --doctest-modules --ignore test/nursery test diff --git a/.github/workflows/tests-xgboost.yaml b/.github/workflows/tests-xgboost.yaml index a8a859dbb1..b41a31f167 100644 --- a/.github/workflows/tests-xgboost.yaml +++ b/.github/workflows/tests-xgboost.yaml @@ -23,10 +23,10 @@ jobs: run: | python -m pip install -U pip setuptools wheel python -m pip install -U "Cython<3.0" - pip install . - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-rotbaum.txt - pip install -r requirements/requirements-rotbaum-extra-methods.txt + pip install . \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-rotbaum.txt \ + -r requirements/requirements-rotbaum-extra-methods.txt - name: Test with pytest run: | pytest -n2 --timeout 120 --doctest-modules test/ext/rotbaum diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a398b822e6..7119f0d83d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,10 +21,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install other dependencies run: | - python -m pip install -U pip - pip install ".[mxnet,arrow,shell]" - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-m-competitions.txt + pip install ".[mxnet,arrow,shell]" \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-m-competitions.txt - name: Test with pytest run: | pytest -n2 --doctest-modules --ignore test/nursery test From 80d043e7c4bb42fd0877bdbc02a9fdda1dea9e6a Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 2 Feb 2024 10:55:16 +0100 Subject: [PATCH 2/6] cleaning `python -m pip` mixing with `pip` --- .github/workflows/mxnet_nightly.yml | 2 +- .github/workflows/pypi.yml | 3 --- .github/workflows/style_type_checks.yml | 1 - .github/workflows/test_release_unix_nightly.yml | 2 +- .github/workflows/tests-prophet.yaml | 1 - .github/workflows/tests-xgboost.yaml | 3 +-- 6 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/mxnet_nightly.yml b/.github/workflows/mxnet_nightly.yml index b95e3d491c..beec24bf6a 100644 --- a/.github/workflows/mxnet_nightly.yml +++ b/.github/workflows/mxnet_nightly.yml @@ -24,7 +24,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install pip==20.2 + pip install pip==20.2 pip install --pre "mxnet<2" -f https://dist.mxnet.io/python pip install "torch==1.10.0" "pytorch-lightning~=1.5" -f https://download.pytorch.org/whl/cpu/torch_stable.html pip install -e ".[shell]" \ diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index cc756c6155..431801fbcf 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -16,9 +16,6 @@ jobs: uses: actions/setup-python@v4 with: python-version: '3.8' - - name: Install dependencies - run: | - python -m pip install setuptools wheel - name: Create `dist` folder run: | python setup.py sdist bdist_wheel diff --git a/.github/workflows/style_type_checks.yml b/.github/workflows/style_type_checks.yml index f8ffcc4ca8..8ad3c0fa68 100644 --- a/.github/workflows/style_type_checks.yml +++ b/.github/workflows/style_type_checks.yml @@ -17,7 +17,6 @@ jobs: python-version: '3.8' - name: Install dependencies run: | - python -m pip install -U pip pip install . pip install click black mypy pip install types-python-dateutil diff --git a/.github/workflows/test_release_unix_nightly.yml b/.github/workflows/test_release_unix_nightly.yml index 84c6746788..b602bc069a 100644 --- a/.github/workflows/test_release_unix_nightly.yml +++ b/.github/workflows/test_release_unix_nightly.yml @@ -23,7 +23,7 @@ jobs: run: | git clone https://github.com/awslabs/gluonts --branch $(curl https://api.github.com/repos/awslabs/gluonts/releases/latest | grep tag_name | cut -d : -f 2,3 | tr -d \"\ | tr -d \,\ ) cd gluon-ts - python -m pip install pip==20.2 + pip install pip==20.2 pip install "mxnet~=1.8.0" "torch==1.10.0" "pytorch-lightning~=1.5" \ -f https://download.pytorch.org/whl/cpu/torch_stable.html pip install -e ".[shell]" \ diff --git a/.github/workflows/tests-prophet.yaml b/.github/workflows/tests-prophet.yaml index c2e7858908..c1d3bdc586 100644 --- a/.github/workflows/tests-prophet.yaml +++ b/.github/workflows/tests-prophet.yaml @@ -21,7 +21,6 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install other dependencies run: | - python -m pip install -U pip pip install . \ -r requirements/requirements-test.txt \ -r requirements/requirements-extras-prophet.txt diff --git a/.github/workflows/tests-xgboost.yaml b/.github/workflows/tests-xgboost.yaml index b41a31f167..30f3a8a08a 100644 --- a/.github/workflows/tests-xgboost.yaml +++ b/.github/workflows/tests-xgboost.yaml @@ -21,8 +21,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install other dependencies run: | - python -m pip install -U pip setuptools wheel - python -m pip install -U "Cython<3.0" + pip install -U "Cython<3.0" numpy # `numpy` is required for scikit-garden pip install . \ -r requirements/requirements-test.txt \ -r requirements/requirements-rotbaum.txt \ From e0dbbdf869161d6e116c85c7eefecd8a1fcb92e3 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 2 Feb 2024 10:59:35 +0100 Subject: [PATCH 3/6] debug installing scikit-garden --- .github/workflows/tests-xgboost.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-xgboost.yaml b/.github/workflows/tests-xgboost.yaml index 30f3a8a08a..8358041550 100644 --- a/.github/workflows/tests-xgboost.yaml +++ b/.github/workflows/tests-xgboost.yaml @@ -21,7 +21,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install other dependencies run: | - pip install -U "Cython<3.0" numpy # `numpy` is required for scikit-garden + pip install "Cython<3.0" numpy scipy scikit-learn # required for scikit-garden pip install . \ -r requirements/requirements-test.txt \ -r requirements/requirements-rotbaum.txt \ From 68ff25f2c95e3e9123ec81f21a3bf6fca1a32b87 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 2 Feb 2024 11:05:55 +0100 Subject: [PATCH 4/6] debug installing scikit-garden --- .github/workflows/tests-xgboost.yaml | 2 +- requirements/requirements-rotbaum-extra-methods.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-xgboost.yaml b/.github/workflows/tests-xgboost.yaml index 8358041550..022b766cf6 100644 --- a/.github/workflows/tests-xgboost.yaml +++ b/.github/workflows/tests-xgboost.yaml @@ -21,7 +21,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install other dependencies run: | - pip install "Cython<3.0" numpy scipy scikit-learn # required for scikit-garden + pip install "Cython<3.0" pip install . \ -r requirements/requirements-test.txt \ -r requirements/requirements-rotbaum.txt \ diff --git a/requirements/requirements-rotbaum-extra-methods.txt b/requirements/requirements-rotbaum-extra-methods.txt index b06879ed3f..d7e312c2e4 100644 --- a/requirements/requirements-rotbaum-extra-methods.txt +++ b/requirements/requirements-rotbaum-extra-methods.txt @@ -1,2 +1,2 @@ lightgbm>=2.3.1 -scikit-garden>=0.1.2 +scikit-garden @ https://github.com/scikit-garden/scikit-garden/archive/refs/heads/master.zip From 47da418820a6339c31024027fcb93df2c258d1fd Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 2 Feb 2024 11:11:16 +0100 Subject: [PATCH 5/6] drop installing scikit-garden --- requirements/requirements-rotbaum-extra-methods.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements-rotbaum-extra-methods.txt b/requirements/requirements-rotbaum-extra-methods.txt index d7e312c2e4..15c5376d9c 100644 --- a/requirements/requirements-rotbaum-extra-methods.txt +++ b/requirements/requirements-rotbaum-extra-methods.txt @@ -1,2 +1,2 @@ lightgbm>=2.3.1 -scikit-garden @ https://github.com/scikit-garden/scikit-garden/archive/refs/heads/master.zip +# scikit-garden @ https://github.com/scikit-garden/scikit-garden/archive/refs/heads/master.zip From 62d44c262a7bf031fc84acfe989b0d774d418762 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Fri, 2 Feb 2024 20:52:41 +0100 Subject: [PATCH 6/6] revert prune in pypi job --- .github/workflows/pypi.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 431801fbcf..d2ca008f29 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -16,6 +16,10 @@ jobs: uses: actions/setup-python@v4 with: python-version: '3.8' + - name: Install dependencies + run: | + python -m pip install -U pip + python -m pip install setuptools wheel - name: Create `dist` folder run: | python setup.py sdist bdist_wheel