Modify test environment pinnings to cover minimum versions (#555)

* Remove black/isort deps as we prefer pre-commit

* Unpin all non python/jdk dependencies

* Minor package corrections for py3.9 jdk11 env

* Set min version constraints for all non-testing dependencies

* Pin all non-test deps for 3.8 testing

* Bump sklearn min version to 1.0.0

* Bump pyarrow min version to 1.0.1

* Fix pip notation for fugue

* Use unpinned deps for cluster testing for now

* Add fugue deps to environments, bump pandas to 1.0.2

* Add back antlr4 version ceiling

* Explicitly mark all fugue dependencies

* Alter test_analyze to avoid rtol

* Bump pandas to 1.0.5 to fix upstream numpy issues

* Alter datetime casting util to dodge panda casting failures

* Bump pandas to 1.1.0 for groupby dropna support

* Simplify string dtype check for get_supported_aggregations

* Add check_dtype=False back to test_group_by_nan

* Bump cluster to python 3.9

* Bump fastapi to 0.69.0, resolve remaining JDBC failures

* Typo - correct pandas version

* Generalize test_multi_case_when's dtype check

* Bump pandas to 1.1.1 to resolve flaky test failures

* Constrain mlflow for windows python 3.8 testing

* Selectors don't work for conda env files

* Problems seem to persist in 1.1.1, bump to 1.1.2

* Remove accidental debug changes

* [test-upstream]

* Use python 3.9 for upstream cluster testing [test-upstream]

* Updated missed pandas pinning

* Unconstrain mlflow to see if Windows failures persist

* Add min version for protobuf

* Bump pyarrow min version to allow for newer protobuf versions

.github/cluster-upstream.yml

@@ -3,7 +3,7 @@ version: '3'
 services:
     dask-scheduler:
         container_name: dask-scheduler
-        image: daskdev/dask:dev
+        image: daskdev/dask:dev-py3.9
         command: dask-scheduler
         environment:
             USE_MAMBA: "true"
@@ -12,10 +12,10 @@ services:
             - "8786:8786"
     dask-worker:
         container_name: dask-worker
-        image: daskdev/dask:dev
+        image: daskdev/dask:dev-py3.9
         command: dask-worker dask-scheduler:8786
         environment:
             USE_MAMBA: "true"
-            EXTRA_CONDA_PACKAGES: "dask/label/dev::dask cloudpickle>=2.1.0 pyarrow>=1.0.0 libstdcxx-ng>=12.1.0"
+            EXTRA_CONDA_PACKAGES: "dask/label/dev::dask cloudpickle>=2.1.0 pyarrow>=3.0.0 libstdcxx-ng>=12.1.0"
         volumes:
             - /tmp:/tmp

.github/cluster.yml

@@ -3,16 +3,16 @@ version: '3'
 services:
     dask-scheduler:
         container_name: dask-scheduler
-        image: daskdev/dask:dev
+        image: daskdev/dask:dev-py3.9
         command: dask-scheduler
         ports:
             - "8786:8786"
     dask-worker:
         container_name: dask-worker
-        image: daskdev/dask:dev
+        image: daskdev/dask:dev-py3.9
         command: dask-worker dask-scheduler:8786
         environment:
             USE_MAMBA: "true"
-            EXTRA_CONDA_PACKAGES: "cloudpickle>=2.1.0 pyarrow>=1.0.0 libstdcxx-ng>=12.1.0"
+            EXTRA_CONDA_PACKAGES: "cloudpickle>=2.1.0 pyarrow>=3.0.0 libstdcxx-ng>=12.1.0"
         volumes:
             - /tmp:/tmp

.github/workflows/test-upstream.yml

@@ -111,11 +111,11 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          python-version: "3.8"
+          python-version: "3.9"
           channel-priority: strict
           channels: dask/label/dev,conda-forge,nodefaults
           activate-environment: dask-sql
-          environment-file: continuous_integration/environment-3.8-jdk11-dev.yaml
+          environment-file: continuous_integration/environment-3.9-jdk11-dev.yaml
       - name: Download the pre-build jar
         uses: actions/download-artifact@v1
         with:

.github/workflows/test.yml

@@ -147,11 +147,11 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          python-version: "3.8"
+          python-version: "3.9"
           channel-priority: strict
           channels: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && 'dask/label/dev,conda-forge,nodefaults' || 'conda-forge,nodefaults' }}
           activate-environment: dask-sql
-          environment-file: continuous_integration/environment-3.8-jdk11-dev.yaml
+          environment-file: continuous_integration/environment-3.9-jdk11-dev.yaml
       - name: Download the pre-build jar
         uses: actions/download-artifact@v1
         with:

continuous_integration/environment-3.10-jdk11-dev.yaml

@@ -3,41 +3,41 @@ channels:
 - conda-forge
 - nodefaults
 dependencies:
-- adagio>=0.2.3
-- antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe
-- black=22.3.0
-- ciso8601>=2.2.0
 - dask-ml>=2022.1.22
 - dask>=2022.3.0
-- fastapi>=0.61.1
-- fs>=2.4.11
+- fastapi>=0.69.0
 - intake>=0.6.0
-- isort=5.7.0
 - jpype1>=1.0.2
 - jsonschema
-- lightgbm>=3.2.1
-- maven>=3.6.0
-- mlflow>=1.19.0
-- mock>=4.0.3
-- nest-asyncio>=1.4.3
+- lightgbm
+- maven
+- mlflow
+- mock
+- nest-asyncio
 - openjdk=11
-- pandas>=1.0.0  # below 1.0, there were no nullable ext. types
-- pip=20.2.4
-- pre-commit>=2.11.1
-- prompt_toolkit>=3.0.8
-- psycopg2>=2.9.1
-- pyarrow>=1.0.0
-- pygments>=2.7.1
-- pyhive>=0.6.4
-- pytest-cov>=2.10.1
+- pandas>=1.1.2
+- pre-commit
+- prompt_toolkit
+- psycopg2
+- pyarrow>=3.0.0
+- pygments
+- pyhive
+- pytest-cov
 - pytest-xdist
-- pytest>=6.0.1
+- pytest
 - python=3.10
-- scikit-learn>=0.24.2
-- sphinx>=3.2.1
-- tpot>=0.11.7
-- triad>=0.5.4
+- scikit-learn>=1.0.0
+- sphinx
+- tpot
 - tzlocal>=2.1
 - uvicorn>=0.11.3
+# fugue dependencies; remove when we conda install fugue
+- adagio
+- antlr4-python3-runtime<4.10
+- ciso8601
+- fs
+- pip
+- qpd
+- triad
 - pip:
   - fugue[sql]>=0.5.3

continuous_integration/environment-3.10-jdk8-dev.yaml

@@ -3,41 +3,41 @@ channels:
 - conda-forge
 - nodefaults
 dependencies:
-- adagio>=0.2.3
-- antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe
-- black=22.3.0
-- ciso8601>=2.2.0
 - dask-ml>=2022.1.22
 - dask>=2022.3.0
-- fastapi>=0.61.1
-- fs>=2.4.11
+- fastapi>=0.69.0
 - intake>=0.6.0
-- isort=5.7.0
 - jpype1>=1.0.2
 - jsonschema
-- lightgbm>=3.2.1
-- maven>=3.6.0
-- mlflow>=1.19.0
-- mock>=4.0.3
-- nest-asyncio>=1.4.3
+- lightgbm
+- maven
+- mlflow
+- mock
+- nest-asyncio
 - openjdk=8
-- pandas>=1.0.0  # below 1.0, there were no nullable ext. types
-- pip=20.2.4
-- pre-commit>=2.11.1
-- prompt_toolkit>=3.0.8
-- psycopg2>=2.9.1
-- pyarrow>=1.0.0
-- pygments>=2.7.1
-- pyhive>=0.6.4
-- pytest-cov>=2.10.1
+- pandas>=1.1.2
+- pre-commit
+- prompt_toolkit
+- psycopg2
+- pyarrow>=3.0.0
+- pygments
+- pyhive
+- pytest-cov
 - pytest-xdist
-- pytest>=6.0.1
+- pytest
 - python=3.10
-- scikit-learn>=0.24.2
-- sphinx>=3.2.1
-- tpot>=0.11.7
-- triad>=0.5.4
+- scikit-learn>=1.0.0
+- sphinx
+- tpot
 - tzlocal>=2.1
 - uvicorn>=0.11.3
+# fugue dependencies; remove when we conda install fugue
+- adagio
+- antlr4-python3-runtime<4.10
+- ciso8601
+- fs
+- pip
+- qpd
+- triad
 - pip:
   - fugue[sql]>=0.5.3

continuous_integration/environment-3.8-jdk11-dev.yaml

@@ -3,41 +3,41 @@ channels:
 - conda-forge
 - nodefaults
 dependencies:
-- adagio>=0.2.3
-- antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe
-- black=22.3.0
-- ciso8601>=2.2.0
-- dask-ml>=2022.1.22
-- dask>=2022.3.0
-- fastapi>=0.61.1
-- fs>=2.4.11
-- intake>=0.6.0
-- isort=5.7.0
-- jpype1>=1.0.2
+- dask-ml=2022.1.22
+- dask=2022.3.0
+- fastapi=0.69.0
+- intake=0.6.0
+- jpype1=1.0.2
 - jsonschema
-- lightgbm>=3.2.1
-- maven>=3.6.0
-- mlflow>=1.19.0
-- mock>=4.0.3
-- nest-asyncio>=1.4.3
+- lightgbm
+- maven
+- mlflow
+- mock
+- nest-asyncio
 - openjdk=11
-- pandas>=1.0.0  # below 1.0, there were no nullable ext. types
-- pip=20.2.4
-- pre-commit>=2.11.1
-- prompt_toolkit>=3.0.8
-- psycopg2>=2.9.1
-- pyarrow>=1.0.0
-- pygments>=2.7.1
-- pyhive>=0.6.4
-- pytest-cov>=2.10.1
+- pandas=1.1.2
+- pre-commit
+- prompt_toolkit
+- psycopg2
+- pyarrow=3.0.0
+- pygments
+- pyhive
+- pytest-cov
 - pytest-xdist
-- pytest>=6.0.1
+- pytest
 - python=3.8
-- scikit-learn>=0.24.2
-- sphinx>=3.2.1
-- tpot>=0.11.7
-- triad>=0.5.4
-- tzlocal>=2.1
-- uvicorn>=0.11.3
+- scikit-learn=1.0.0
+- sphinx
+- tpot
+- tzlocal=2.1
+- uvicorn=0.11.3
+# fugue dependencies; remove when we conda install fugue
+- adagio
+- antlr4-python3-runtime<4.10
+- ciso8601
+- fs
+- pip
+- qpd
+- triad
 - pip:
-  - fugue[sql]>=0.5.3
+  - fugue[sql]==0.5.3

continuous_integration/environment-3.8-jdk8-dev.yaml

@@ -3,41 +3,41 @@ channels:
 - conda-forge
 - nodefaults
 dependencies:
-- adagio>=0.2.3
-- antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe
-- black=22.3.0
-- ciso8601>=2.2.0
-- dask-ml>=2022.1.22
-- dask>=2022.3.0
-- fastapi>=0.61.1
-- fs>=2.4.11
-- intake>=0.6.0
-- isort=5.7.0
-- jpype1>=1.0.2
+- dask-ml=2022.1.22
+- dask=2022.3.0
+- fastapi=0.69.0
+- intake=0.6.0
+- jpype1=1.0.2
 - jsonschema
-- lightgbm>=3.2.1
-- maven>=3.6.0
-- mlflow>=1.19.0
-- mock>=4.0.3
-- nest-asyncio>=1.4.3
+- lightgbm
+- maven
+- mlflow
+- mock
+- nest-asyncio
 - openjdk=8
-- pandas>=1.0.0  # below 1.0, there were no nullable ext. types
-- pip=20.2.4
-- pre-commit>=2.11.1
-- prompt_toolkit>=3.0.8
-- psycopg2>=2.9.1
-- pyarrow>=1.0.0
-- pygments>=2.7.1
-- pyhive>=0.6.4
-- pytest-cov>=2.10.1
+- pandas=1.1.2
+- pre-commit
+- prompt_toolkit
+- psycopg2
+- pyarrow=3.0.0
+- pygments
+- pyhive
+- pytest-cov
 - pytest-xdist
-- pytest>=6.0.1
+- pytest
 - python=3.8
-- scikit-learn>=0.24.2
-- sphinx>=3.2.1
-- tpot>=0.11.7
-- triad>=0.5.4
-- tzlocal>=2.1
-- uvicorn>=0.11.3
+- scikit-learn=1.0.0
+- sphinx
+- tpot
+- tzlocal=2.1
+- uvicorn=0.11.3
+# fugue dependencies; remove when we conda install fugue
+- adagio
+- antlr4-python3-runtime<4.10
+- ciso8601
+- fs
+- pip
+- qpd
+- triad
 - pip:
-  - fugue[sql]>=0.5.3
+  - fugue[sql]==0.5.3