From c49dd9bf7b424d14965111fe150bcb0683ea5fe8 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 26 May 2022 12:38:39 -0700 Subject: [PATCH 01/33] Remove black/isort deps as we prefer pre-commit --- continuous_integration/environment-3.10-jdk11-dev.yaml | 2 -- continuous_integration/environment-3.10-jdk8-dev.yaml | 2 -- continuous_integration/environment-3.8-jdk11-dev.yaml | 2 -- continuous_integration/environment-3.8-jdk8-dev.yaml | 2 -- continuous_integration/environment-3.9-jdk11-dev.yaml | 2 -- continuous_integration/environment-3.9-jdk8-dev.yaml | 2 -- 6 files changed, 12 deletions(-) diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index d04d81130..b0d5b9764 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -5,14 +5,12 @@ channels: dependencies: - adagio>=0.2.3 - antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe -- black=22.3.0 - ciso8601>=2.2.0 - dask-ml>=2022.1.22 - dask>=2022.3.0 - fastapi>=0.61.1 - fs>=2.4.11 - intake>=0.6.0 -- isort=5.7.0 - jpype1>=1.0.2 - jsonschema - lightgbm>=3.2.1 diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index 4a56a9ccf..e0d27270c 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -5,14 +5,12 @@ channels: dependencies: - adagio>=0.2.3 - antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe -- black=22.3.0 - ciso8601>=2.2.0 - dask-ml>=2022.1.22 - dask>=2022.3.0 - fastapi>=0.61.1 - fs>=2.4.11 - intake>=0.6.0 -- isort=5.7.0 - jpype1>=1.0.2 - jsonschema - lightgbm>=3.2.1 diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index 0e0ef3d7d..9d50a690f 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -5,14 +5,12 @@ channels: dependencies: - adagio>=0.2.3 - antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe -- black=22.3.0 - ciso8601>=2.2.0 - dask-ml>=2022.1.22 - dask>=2022.3.0 - fastapi>=0.61.1 - fs>=2.4.11 - intake>=0.6.0 -- isort=5.7.0 - jpype1>=1.0.2 - jsonschema - lightgbm>=3.2.1 diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index 9f225bed5..d6c6db066 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -5,14 +5,12 @@ channels: dependencies: - adagio>=0.2.3 - antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe -- black=22.3.0 - ciso8601>=2.2.0 - dask-ml>=2022.1.22 - dask>=2022.3.0 - fastapi>=0.61.1 - fs>=2.4.11 - intake>=0.6.0 -- isort=5.7.0 - jpype1>=1.0.2 - jsonschema - lightgbm>=3.2.1 diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index 1c368fbbd..23cc43276 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -5,10 +5,8 @@ channels: dependencies: - adagio>=0.2.3 - antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe -- black=22.3.0 - ciso8601>=2.2.0 - dask-ml>=2022.1.22 -- dask>=2022.3.0 - fastapi>=0.61.1 - fs>=2.4.11 - intake>=0.6.0 diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index c0dbe1cae..f79410d64 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -5,14 +5,12 @@ channels: dependencies: - adagio>=0.2.3 - antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe -- black=22.3.0 - ciso8601>=2.2.0 - dask-ml>=2022.1.22 - dask>=2022.3.0 - fastapi>=0.61.1 - fs>=2.4.11 - intake>=0.6.0 -- isort=5.7.0 - jpype1>=1.0.2 - jsonschema - lightgbm>=3.2.1 From a3f10d60d1cb1f6a26164242ab891281f640e371 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 26 May 2022 12:44:29 -0700 Subject: [PATCH 02/33] Unpin all non python/jdk dependencies --- .../environment-3.10-jdk11-dev.yaml | 62 +++++++++---------- .../environment-3.10-jdk8-dev.yaml | 62 +++++++++---------- .../environment-3.8-jdk11-dev.yaml | 62 +++++++++---------- .../environment-3.8-jdk8-dev.yaml | 62 +++++++++---------- .../environment-3.9-jdk11-dev.yaml | 62 +++++++++---------- .../environment-3.9-jdk8-dev.yaml | 62 +++++++++---------- 6 files changed, 186 insertions(+), 186 deletions(-) diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index b0d5b9764..1b1262a29 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -3,39 +3,39 @@ channels: - conda-forge - nodefaults dependencies: -- adagio>=0.2.3 -- antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe -- ciso8601>=2.2.0 -- dask-ml>=2022.1.22 -- dask>=2022.3.0 -- fastapi>=0.61.1 -- fs>=2.4.11 -- intake>=0.6.0 -- jpype1>=1.0.2 +- adagio +- antlr4-python3-runtime +- ciso8601 +- dask-ml +- dask +- fastapi +- fs +- intake +- jpype1 - jsonschema -- lightgbm>=3.2.1 -- maven>=3.6.0 -- mlflow>=1.19.0 -- mock>=4.0.3 -- nest-asyncio>=1.4.3 +- lightgbm +- maven +- mlflow +- mock +- nest-asyncio - openjdk=11 -- pandas>=1.0.0 # below 1.0, there were no nullable ext. types -- pip=20.2.4 -- pre-commit>=2.11.1 -- prompt_toolkit>=3.0.8 -- psycopg2>=2.9.1 -- pyarrow>=1.0.0 -- pygments>=2.7.1 -- pyhive>=0.6.4 -- pytest-cov>=2.10.1 +- pandas +- pip +- pre-commit +- prompt_toolkit +- psycopg2 +- pyarrow +- pygments +- pyhive +- pytest-cov - pytest-xdist -- pytest>=6.0.1 +- pytest - python=3.10 -- scikit-learn>=0.24.2 -- sphinx>=3.2.1 -- tpot>=0.11.7 -- triad>=0.5.4 -- tzlocal>=2.1 -- uvicorn>=0.11.3 +- scikit-learn +- sphinx +- tpot +- triad +- tzlocal +- uvicorn - pip: - - fugue[sql]>=0.5.3 + - fugue[sql] diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index e0d27270c..425637298 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -3,39 +3,39 @@ channels: - conda-forge - nodefaults dependencies: -- adagio>=0.2.3 -- antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe -- ciso8601>=2.2.0 -- dask-ml>=2022.1.22 -- dask>=2022.3.0 -- fastapi>=0.61.1 -- fs>=2.4.11 -- intake>=0.6.0 -- jpype1>=1.0.2 +- adagio +- antlr4-python3-runtime +- ciso8601 +- dask-ml +- dask +- fastapi +- fs +- intake +- jpype1 - jsonschema -- lightgbm>=3.2.1 -- maven>=3.6.0 -- mlflow>=1.19.0 -- mock>=4.0.3 -- nest-asyncio>=1.4.3 +- lightgbm +- maven +- mlflow +- mock +- nest-asyncio - openjdk=8 -- pandas>=1.0.0 # below 1.0, there were no nullable ext. types -- pip=20.2.4 -- pre-commit>=2.11.1 -- prompt_toolkit>=3.0.8 -- psycopg2>=2.9.1 -- pyarrow>=1.0.0 -- pygments>=2.7.1 -- pyhive>=0.6.4 -- pytest-cov>=2.10.1 +- pandas +- pip +- pre-commit +- prompt_toolkit +- psycopg2 +- pyarrow +- pygments +- pyhive +- pytest-cov - pytest-xdist -- pytest>=6.0.1 +- pytest - python=3.10 -- scikit-learn>=0.24.2 -- sphinx>=3.2.1 -- tpot>=0.11.7 -- triad>=0.5.4 -- tzlocal>=2.1 -- uvicorn>=0.11.3 +- scikit-learn +- sphinx +- tpot +- triad +- tzlocal +- uvicorn - pip: - - fugue[sql]>=0.5.3 + - fugue[sql] diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index 9d50a690f..37d44e284 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -3,39 +3,39 @@ channels: - conda-forge - nodefaults dependencies: -- adagio>=0.2.3 -- antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe -- ciso8601>=2.2.0 -- dask-ml>=2022.1.22 -- dask>=2022.3.0 -- fastapi>=0.61.1 -- fs>=2.4.11 -- intake>=0.6.0 -- jpype1>=1.0.2 +- adagio +- antlr4-python3-runtime +- ciso8601 +- dask-ml +- dask +- fastapi +- fs +- intake +- jpype1 - jsonschema -- lightgbm>=3.2.1 -- maven>=3.6.0 -- mlflow>=1.19.0 -- mock>=4.0.3 -- nest-asyncio>=1.4.3 +- lightgbm +- maven +- mlflow +- mock +- nest-asyncio - openjdk=11 -- pandas>=1.0.0 # below 1.0, there were no nullable ext. types -- pip=20.2.4 -- pre-commit>=2.11.1 -- prompt_toolkit>=3.0.8 -- psycopg2>=2.9.1 -- pyarrow>=1.0.0 -- pygments>=2.7.1 -- pyhive>=0.6.4 -- pytest-cov>=2.10.1 +- pandas +- pip +- pre-commit +- prompt_toolkit +- psycopg2 +- pyarrow +- pygments +- pyhive +- pytest-cov - pytest-xdist -- pytest>=6.0.1 +- pytest - python=3.8 -- scikit-learn>=0.24.2 -- sphinx>=3.2.1 -- tpot>=0.11.7 -- triad>=0.5.4 -- tzlocal>=2.1 -- uvicorn>=0.11.3 +- scikit-learn +- sphinx +- tpot +- triad +- tzlocal +- uvicorn - pip: - - fugue[sql]>=0.5.3 + - fugue[sql] diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index d6c6db066..4820e3461 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -3,39 +3,39 @@ channels: - conda-forge - nodefaults dependencies: -- adagio>=0.2.3 -- antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe -- ciso8601>=2.2.0 -- dask-ml>=2022.1.22 -- dask>=2022.3.0 -- fastapi>=0.61.1 -- fs>=2.4.11 -- intake>=0.6.0 -- jpype1>=1.0.2 +- adagio +- antlr4-python3-runtime +- ciso8601 +- dask-ml +- dask +- fastapi +- fs +- intake +- jpype1 - jsonschema -- lightgbm>=3.2.1 -- maven>=3.6.0 -- mlflow>=1.19.0 -- mock>=4.0.3 -- nest-asyncio>=1.4.3 +- lightgbm +- maven +- mlflow +- mock +- nest-asyncio - openjdk=8 -- pandas>=1.0.0 # below 1.0, there were no nullable ext. types -- pip=20.2.4 -- pre-commit>=2.11.1 -- prompt_toolkit>=3.0.8 -- psycopg2>=2.9.1 -- pyarrow>=1.0.0 -- pygments>=2.7.1 -- pyhive>=0.6.4 -- pytest-cov>=2.10.1 +- pandas +- pip +- pre-commit +- prompt_toolkit +- psycopg2 +- pyarrow +- pygments +- pyhive +- pytest-cov - pytest-xdist -- pytest>=6.0.1 +- pytest - python=3.8 -- scikit-learn>=0.24.2 -- sphinx>=3.2.1 -- tpot>=0.11.7 -- triad>=0.5.4 -- tzlocal>=2.1 -- uvicorn>=0.11.3 +- scikit-learn +- sphinx +- tpot +- triad +- tzlocal +- uvicorn - pip: - - fugue[sql]>=0.5.3 + - fugue[sql] diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index 23cc43276..d2b368e6e 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -3,40 +3,40 @@ channels: - conda-forge - nodefaults dependencies: -- adagio>=0.2.3 -- antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe -- ciso8601>=2.2.0 -- dask-ml>=2022.1.22 -- fastapi>=0.61.1 -- fs>=2.4.11 -- intake>=0.6.0 +- adagio +- antlr4-python3-runtime +- ciso8601 +- dask-ml +- fastapi +- fs +- intake - isort=5.7.0 -- jpype1>=1.0.2 +- jpype1 - jsonschema -- lightgbm>=3.2.1 -- maven>=3.6.0 -- mlflow>=1.19.0 -- mock>=4.0.3 -- nest-asyncio>=1.4.3 +- lightgbm +- maven +- mlflow +- mock +- nest-asyncio - openjdk=11 -- pandas>=1.0.0 # below 1.0, there were no nullable ext. types -- pip=20.2.4 -- pre-commit>=2.11.1 -- prompt_toolkit>=3.0.8 -- psycopg2>=2.9.1 -- pyarrow>=1.0.0 -- pygments>=2.7.1 -- pyhive>=0.6.4 -- pytest-cov>=2.10.1 +- pandas +- pip +- pre-commit +- prompt_toolkit +- psycopg2 +- pyarrow +- pygments +- pyhive +- pytest-cov - pytest-xdist -- pytest>=6.0.1 +- pytest - python=3.9 -- scikit-learn>=0.24.2 -- sphinx>=3.2.1 -- tpot>=0.11.7 -- triad>=0.5.4 -- tzlocal>=2.1 -- uvicorn>=0.11.3 -- cfn-lint>=0.4.0 +- scikit-learn +- sphinx +- tpot +- triad +- tzlocal +- uvicorn +- cfn-lint - pip: - - fugue[sql]>=0.5.3 + - fugue[sql] diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index f79410d64..049bce223 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -3,39 +3,39 @@ channels: - conda-forge - nodefaults dependencies: -- adagio>=0.2.3 -- antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe -- ciso8601>=2.2.0 -- dask-ml>=2022.1.22 -- dask>=2022.3.0 -- fastapi>=0.61.1 -- fs>=2.4.11 -- intake>=0.6.0 -- jpype1>=1.0.2 +- adagio +- antlr4-python3-runtime +- ciso8601 +- dask-ml +- dask +- fastapi +- fs +- intake +- jpype1 - jsonschema -- lightgbm>=3.2.1 -- maven>=3.6.0 -- mlflow>=1.19.0 -- mock>=4.0.3 -- nest-asyncio>=1.4.3 +- lightgbm +- maven +- mlflow +- mock +- nest-asyncio - openjdk=8 -- pandas>=1.0.0 # below 1.0, there were no nullable ext. types -- pip=20.2.4 -- pre-commit>=2.11.1 -- prompt_toolkit>=3.0.8 -- psycopg2>=2.9.1 -- pyarrow>=1.0.0 -- pygments>=2.7.1 -- pyhive>=0.6.4 -- pytest-cov>=2.10.1 +- pandas +- pip +- pre-commit +- prompt_toolkit +- psycopg2 +- pyarrow +- pygments +- pyhive +- pytest-cov - pytest-xdist -- pytest>=6.0.1 +- pytest - python=3.9 -- scikit-learn>=0.24.2 -- sphinx>=3.2.1 -- tpot>=0.11.7 -- triad>=0.5.4 -- tzlocal>=2.1 -- uvicorn>=0.11.3 +- scikit-learn +- sphinx +- tpot +- triad +- tzlocal +- uvicorn - pip: - - fugue[sql]>=0.5.3 + - fugue[sql] From 9463b53a4ac5892a0cf22afeb336100622dd7e2b Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 26 May 2022 12:46:11 -0700 Subject: [PATCH 03/33] Minor package corrections for py3.9 jdk11 env --- continuous_integration/environment-3.9-jdk11-dev.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index d2b368e6e..e8bab0e93 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -7,10 +7,10 @@ dependencies: - antlr4-python3-runtime - ciso8601 - dask-ml +- dask - fastapi - fs - intake -- isort=5.7.0 - jpype1 - jsonschema - lightgbm @@ -37,6 +37,5 @@ dependencies: - triad - tzlocal - uvicorn -- cfn-lint - pip: - fugue[sql] From 4dcffe112f8bcb85ceed78b55cef6961d322a8d9 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 26 May 2022 12:55:28 -0700 Subject: [PATCH 04/33] Set min version constraints for all non-testing dependencies --- .../environment-3.10-jdk11-dev.yaml | 22 +++++++++---------- .../environment-3.10-jdk8-dev.yaml | 22 +++++++++---------- .../environment-3.8-jdk11-dev.yaml | 22 +++++++++---------- .../environment-3.8-jdk8-dev.yaml | 22 +++++++++---------- .../environment-3.9-jdk11-dev.yaml | 22 +++++++++---------- .../environment-3.9-jdk8-dev.yaml | 22 +++++++++---------- 6 files changed, 66 insertions(+), 66 deletions(-) diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index 1b1262a29..dbf3b589a 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -6,12 +6,12 @@ dependencies: - adagio - antlr4-python3-runtime - ciso8601 -- dask-ml -- dask -- fastapi +- dask-ml>=2022.1.22 +- dask>=2022.3.0 +- fastapi>=0.61.1 - fs -- intake -- jpype1 +- intake>=0.6.0 +- jpype1>=1.0.2 - jsonschema - lightgbm - maven @@ -19,23 +19,23 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas +- pandas>=1.0.0 - pip - pre-commit - prompt_toolkit - psycopg2 -- pyarrow +- pyarrow>=1.0.0 - pygments - pyhive - pytest-cov - pytest-xdist - pytest - python=3.10 -- scikit-learn +- scikit-learn>=0.24.2 - sphinx - tpot - triad -- tzlocal -- uvicorn +- tzlocal>=2.1 +- uvicorn>=0.11.3 - pip: - - fugue[sql] + - fugue[sql]>=0.5.3 diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index 425637298..70bb93c4f 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -6,12 +6,12 @@ dependencies: - adagio - antlr4-python3-runtime - ciso8601 -- dask-ml -- dask -- fastapi +- dask-ml>=2022.1.22 +- dask>=2022.3.0 +- fastapi>=0.61.1 - fs -- intake -- jpype1 +- intake>=0.6.0 +- jpype1>=1.0.2 - jsonschema - lightgbm - maven @@ -19,23 +19,23 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas +- pandas>=1.0.0 - pip - pre-commit - prompt_toolkit - psycopg2 -- pyarrow +- pyarrow>=1.0.0 - pygments - pyhive - pytest-cov - pytest-xdist - pytest - python=3.10 -- scikit-learn +- scikit-learn>=0.24.2 - sphinx - tpot - triad -- tzlocal -- uvicorn +- tzlocal>=2.1 +- uvicorn>=0.11.3 - pip: - - fugue[sql] + - fugue[sql]>=0.5.3 diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index 37d44e284..6ad529419 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -6,12 +6,12 @@ dependencies: - adagio - antlr4-python3-runtime - ciso8601 -- dask-ml -- dask -- fastapi +- dask-ml>=2022.1.22 +- dask>=2022.3.0 +- fastapi>=0.61.1 - fs -- intake -- jpype1 +- intake>=0.6.0 +- jpype1>=1.0.2 - jsonschema - lightgbm - maven @@ -19,23 +19,23 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas +- pandas>=1.0.0 - pip - pre-commit - prompt_toolkit - psycopg2 -- pyarrow +- pyarrow>=1.0.0 - pygments - pyhive - pytest-cov - pytest-xdist - pytest - python=3.8 -- scikit-learn +- scikit-learn>=0.24.2 - sphinx - tpot - triad -- tzlocal -- uvicorn +- tzlocal>=2.1 +- uvicorn>=0.11.3 - pip: - - fugue[sql] + - fugue[sql]>=0.5.3 diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index 4820e3461..8e9606c23 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -6,12 +6,12 @@ dependencies: - adagio - antlr4-python3-runtime - ciso8601 -- dask-ml -- dask -- fastapi +- dask-ml>=2022.1.22 +- dask>=2022.3.0 +- fastapi>=0.61.1 - fs -- intake -- jpype1 +- intake>=0.6.0 +- jpype1>=1.0.2 - jsonschema - lightgbm - maven @@ -19,23 +19,23 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas +- pandas>=1.0.0 - pip - pre-commit - prompt_toolkit - psycopg2 -- pyarrow +- pyarrow>=1.0.0 - pygments - pyhive - pytest-cov - pytest-xdist - pytest - python=3.8 -- scikit-learn +- scikit-learn>=0.24.2 - sphinx - tpot - triad -- tzlocal -- uvicorn +- tzlocal>=2.1 +- uvicorn>=0.11.3 - pip: - - fugue[sql] + - fugue[sql]>=0.5.3 diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index e8bab0e93..d815f9894 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -6,12 +6,12 @@ dependencies: - adagio - antlr4-python3-runtime - ciso8601 -- dask-ml -- dask -- fastapi +- dask-ml>=2022.1.22 +- dask>=2022.3.0 +- fastapi>=0.61.1 - fs -- intake -- jpype1 +- intake>=0.6.0 +- jpype1>=1.0.2 - jsonschema - lightgbm - maven @@ -19,23 +19,23 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas +- pandas>=1.0.0 - pip - pre-commit - prompt_toolkit - psycopg2 -- pyarrow +- pyarrow>=1.0.0 - pygments - pyhive - pytest-cov - pytest-xdist - pytest - python=3.9 -- scikit-learn +- scikit-learn>=0.24.2 - sphinx - tpot - triad -- tzlocal -- uvicorn +- tzlocal>=2.1 +- uvicorn>=0.11.3 - pip: - - fugue[sql] + - fugue[sql]>=0.5.3 diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index 049bce223..8de1627cd 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -6,12 +6,12 @@ dependencies: - adagio - antlr4-python3-runtime - ciso8601 -- dask-ml -- dask -- fastapi +- dask-ml>=2022.1.22 +- dask>=2022.3.0 +- fastapi>=0.61.1 - fs -- intake -- jpype1 +- intake>=0.6.0 +- jpype1>=1.0.2 - jsonschema - lightgbm - maven @@ -19,23 +19,23 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas +- pandas>=1.0.0 - pip - pre-commit - prompt_toolkit - psycopg2 -- pyarrow +- pyarrow>=1.0.0 - pygments - pyhive - pytest-cov - pytest-xdist - pytest - python=3.9 -- scikit-learn +- scikit-learn>=0.24.2 - sphinx - tpot - triad -- tzlocal -- uvicorn +- tzlocal>=2.1 +- uvicorn>=0.11.3 - pip: - - fugue[sql] + - fugue[sql]>=0.5.3 From 514e3fa23070cbfd5ab430fe1f39e2c5bc4081bf Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 26 May 2022 12:56:13 -0700 Subject: [PATCH 05/33] Pin all non-test deps for 3.8 testing --- .../environment-3.8-jdk11-dev.yaml | 22 +++++++++---------- .../environment-3.8-jdk8-dev.yaml | 22 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index 6ad529419..4be199247 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -6,12 +6,12 @@ dependencies: - adagio - antlr4-python3-runtime - ciso8601 -- dask-ml>=2022.1.22 -- dask>=2022.3.0 -- fastapi>=0.61.1 +- dask-ml=2022.1.22 +- dask=2022.3.0 +- fastapi=0.61.1 - fs -- intake>=0.6.0 -- jpype1>=1.0.2 +- intake=0.6.0 +- jpype1=1.0.2 - jsonschema - lightgbm - maven @@ -19,23 +19,23 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas>=1.0.0 +- pandas=1.0.0 - pip - pre-commit - prompt_toolkit - psycopg2 -- pyarrow>=1.0.0 +- pyarrow=1.0.0 - pygments - pyhive - pytest-cov - pytest-xdist - pytest - python=3.8 -- scikit-learn>=0.24.2 +- scikit-learn=0.24.2 - sphinx - tpot - triad -- tzlocal>=2.1 -- uvicorn>=0.11.3 +- tzlocal=2.1 +- uvicorn=0.11.3 - pip: - - fugue[sql]>=0.5.3 + - fugue[sql]=0.5.3 diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index 8e9606c23..96ed0b446 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -6,12 +6,12 @@ dependencies: - adagio - antlr4-python3-runtime - ciso8601 -- dask-ml>=2022.1.22 -- dask>=2022.3.0 -- fastapi>=0.61.1 +- dask-ml=2022.1.22 +- dask=2022.3.0 +- fastapi=0.61.1 - fs -- intake>=0.6.0 -- jpype1>=1.0.2 +- intake=0.6.0 +- jpype1=1.0.2 - jsonschema - lightgbm - maven @@ -19,23 +19,23 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas>=1.0.0 +- pandas=1.0.0 - pip - pre-commit - prompt_toolkit - psycopg2 -- pyarrow>=1.0.0 +- pyarrow=1.0.0 - pygments - pyhive - pytest-cov - pytest-xdist - pytest - python=3.8 -- scikit-learn>=0.24.2 +- scikit-learn=0.24.2 - sphinx - tpot - triad -- tzlocal>=2.1 -- uvicorn>=0.11.3 +- tzlocal=2.1 +- uvicorn=0.11.3 - pip: - - fugue[sql]>=0.5.3 + - fugue[sql]=0.5.3 From f089305d601d2faa13ffcd7968edeca6d76f7e04 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 26 May 2022 13:20:32 -0700 Subject: [PATCH 06/33] Bump sklearn min version to 1.0.0 --- continuous_integration/environment-3.10-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.10-jdk8-dev.yaml | 2 +- continuous_integration/environment-3.8-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.8-jdk8-dev.yaml | 2 +- continuous_integration/environment-3.9-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.9-jdk8-dev.yaml | 2 +- docker/conda.txt | 2 +- docker/main.dockerfile | 2 +- setup.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index dbf3b589a..7309fb920 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -31,7 +31,7 @@ dependencies: - pytest-xdist - pytest - python=3.10 -- scikit-learn>=0.24.2 +- scikit-learn>=1.0.0 - sphinx - tpot - triad diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index 70bb93c4f..f33ee670c 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -31,7 +31,7 @@ dependencies: - pytest-xdist - pytest - python=3.10 -- scikit-learn>=0.24.2 +- scikit-learn>=1.0.0 - sphinx - tpot - triad diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index 4be199247..ffb499e4f 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -31,7 +31,7 @@ dependencies: - pytest-xdist - pytest - python=3.8 -- scikit-learn=0.24.2 +- scikit-learn=1.0.0 - sphinx - tpot - triad diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index 96ed0b446..36af50a23 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -31,7 +31,7 @@ dependencies: - pytest-xdist - pytest - python=3.8 -- scikit-learn=0.24.2 +- scikit-learn=1.0.0 - sphinx - tpot - triad diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index d815f9894..47c9c1685 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -31,7 +31,7 @@ dependencies: - pytest-xdist - pytest - python=3.9 -- scikit-learn>=0.24.2 +- scikit-learn>=1.0.0 - sphinx - tpot - triad diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index 8de1627cd..48979a366 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -31,7 +31,7 @@ dependencies: - pytest-xdist - pytest - python=3.9 -- scikit-learn>=0.24.2 +- scikit-learn>=1.0.0 - sphinx - tpot - triad diff --git a/docker/conda.txt b/docker/conda.txt index e40d1069d..d63351af0 100644 --- a/docker/conda.txt +++ b/docker/conda.txt @@ -17,7 +17,7 @@ pyarrow>=1.0.0 prompt_toolkit>=3.0.8 pygments>=2.7.1 dask-ml>=2022.1.22 -scikit-learn>=0.24.2 +scikit-learn>=1.0.0 intake>=0.6.0 pre-commit>=2.11.1 black=22.3.0 diff --git a/docker/main.dockerfile b/docker/main.dockerfile index 848948dd5..17ed4b691 100644 --- a/docker/main.dockerfile +++ b/docker/main.dockerfile @@ -17,7 +17,7 @@ RUN conda config --add channels conda-forge \ "prompt_toolkit>=3.0.8" \ "pygments>=2.7.1" \ "dask-ml>=2022.1.22" \ - "scikit-learn>=0.24.2" \ + "scikit-learn>=1.0.0" \ "intake>=0.6.0" \ && conda clean -ay diff --git a/setup.py b/setup.py index be50794e1..aa8fa03e4 100755 --- a/setup.py +++ b/setup.py @@ -106,7 +106,7 @@ def build(self): "sphinx>=3.2.1", "pyarrow>=1.0.0", "dask-ml>=2022.1.22", - "scikit-learn>=0.24.2", + "scikit-learn>=1.0.0", "intake>=0.6.0", "pre-commit", "black==22.3.0", From 80224308ab1d44fa9150b3c585d6ef3b608a74ca Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 26 May 2022 13:51:43 -0700 Subject: [PATCH 07/33] Bump pyarrow min version to 1.0.1 --- .github/cluster-upstream.yml | 2 +- .github/cluster.yml | 2 +- continuous_integration/environment-3.10-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.10-jdk8-dev.yaml | 2 +- continuous_integration/environment-3.8-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.8-jdk8-dev.yaml | 2 +- continuous_integration/environment-3.9-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.9-jdk8-dev.yaml | 2 +- docker/conda.txt | 2 +- docker/main.dockerfile | 2 +- setup.py | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/cluster-upstream.yml b/.github/cluster-upstream.yml index db864bedc..e74fa9686 100644 --- a/.github/cluster-upstream.yml +++ b/.github/cluster-upstream.yml @@ -13,6 +13,6 @@ services: command: dask-worker dask-scheduler:8786 environment: USE_MAMBA: "true" - EXTRA_CONDA_PACKAGES: "dask/label/dev::dask cloudpickle>=2.1.0 pyarrow>=1.0.0 libstdcxx-ng>=12.1.0" + EXTRA_CONDA_PACKAGES: "dask/label/dev::dask cloudpickle>=2.1.0 pyarrow>=1.0.1 libstdcxx-ng>=12.1.0" volumes: - /tmp:/tmp diff --git a/.github/cluster.yml b/.github/cluster.yml index d59624c7b..24f75a35b 100644 --- a/.github/cluster.yml +++ b/.github/cluster.yml @@ -13,6 +13,6 @@ services: command: dask-worker dask-scheduler:8786 environment: USE_MAMBA: "true" - EXTRA_CONDA_PACKAGES: "cloudpickle>=2.1.0 pyarrow>=1.0.0 libstdcxx-ng>=12.1.0" + EXTRA_CONDA_PACKAGES: "cloudpickle>=2.1.0 pyarrow>=1.0.1 libstdcxx-ng>=12.1.0" volumes: - /tmp:/tmp diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index 7309fb920..6aa63263a 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -24,7 +24,7 @@ dependencies: - pre-commit - prompt_toolkit - psycopg2 -- pyarrow>=1.0.0 +- pyarrow>=1.0.1 - pygments - pyhive - pytest-cov diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index f33ee670c..077d395c6 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -24,7 +24,7 @@ dependencies: - pre-commit - prompt_toolkit - psycopg2 -- pyarrow>=1.0.0 +- pyarrow>=1.0.1 - pygments - pyhive - pytest-cov diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index ffb499e4f..bc4cac0d6 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -24,7 +24,7 @@ dependencies: - pre-commit - prompt_toolkit - psycopg2 -- pyarrow=1.0.0 +- pyarrow=1.0.1 - pygments - pyhive - pytest-cov diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index 36af50a23..0e1cd3838 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -24,7 +24,7 @@ dependencies: - pre-commit - prompt_toolkit - psycopg2 -- pyarrow=1.0.0 +- pyarrow=1.0.1 - pygments - pyhive - pytest-cov diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index 47c9c1685..83242f420 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -24,7 +24,7 @@ dependencies: - pre-commit - prompt_toolkit - psycopg2 -- pyarrow>=1.0.0 +- pyarrow>=1.0.1 - pygments - pyhive - pytest-cov diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index 48979a366..274ebbea7 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -24,7 +24,7 @@ dependencies: - pre-commit - prompt_toolkit - psycopg2 -- pyarrow>=1.0.0 +- pyarrow>=1.0.1 - pygments - pyhive - pytest-cov diff --git a/docker/conda.txt b/docker/conda.txt index d63351af0..0382e77ca 100644 --- a/docker/conda.txt +++ b/docker/conda.txt @@ -13,7 +13,7 @@ tzlocal>=2.1 fastapi>=0.61.1 nest-asyncio>=1.4.3 uvicorn>=0.11.3 -pyarrow>=1.0.0 +pyarrow>=1.0.1 prompt_toolkit>=3.0.8 pygments>=2.7.1 dask-ml>=2022.1.22 diff --git a/docker/main.dockerfile b/docker/main.dockerfile index 17ed4b691..297ad710b 100644 --- a/docker/main.dockerfile +++ b/docker/main.dockerfile @@ -13,7 +13,7 @@ RUN conda config --add channels conda-forge \ "tzlocal>=2.1" \ "fastapi>=0.61.1" \ "uvicorn>=0.11.3" \ - "pyarrow>=1.0.0" \ + "pyarrow>=1.0.1" \ "prompt_toolkit>=3.0.8" \ "pygments>=2.7.1" \ "dask-ml>=2022.1.22" \ diff --git a/setup.py b/setup.py index aa8fa03e4..6be1107dc 100755 --- a/setup.py +++ b/setup.py @@ -104,7 +104,7 @@ def build(self): "pytest-cov>=2.10.1", "mock>=4.0.3", "sphinx>=3.2.1", - "pyarrow>=1.0.0", + "pyarrow>=1.0.1", "dask-ml>=2022.1.22", "scikit-learn>=1.0.0", "intake>=0.6.0", From e17af8eed7761d655121a0db8db8c16c597a6258 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 26 May 2022 13:55:25 -0700 Subject: [PATCH 08/33] Fix pip notation for fugue --- continuous_integration/environment-3.8-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.8-jdk8-dev.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index bc4cac0d6..ecd9b99da 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -38,4 +38,4 @@ dependencies: - tzlocal=2.1 - uvicorn=0.11.3 - pip: - - fugue[sql]=0.5.3 + - fugue[sql]==0.5.3 diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index 0e1cd3838..6d518e3f2 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -38,4 +38,4 @@ dependencies: - tzlocal=2.1 - uvicorn=0.11.3 - pip: - - fugue[sql]=0.5.3 + - fugue[sql]==0.5.3 From 23acd298e0393b5bf3e7cc2439db731d1bed8250 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 26 May 2022 14:14:48 -0700 Subject: [PATCH 09/33] Use unpinned deps for cluster testing for now --- .github/workflows/test-upstream.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-upstream.yml b/.github/workflows/test-upstream.yml index e2d0577bf..7f41a4d6c 100644 --- a/.github/workflows/test-upstream.yml +++ b/.github/workflows/test-upstream.yml @@ -111,11 +111,11 @@ jobs: with: miniforge-variant: Mambaforge use-mamba: true - python-version: "3.8" + python-version: "3.9" channel-priority: strict channels: dask/label/dev,conda-forge,nodefaults activate-environment: dask-sql - environment-file: continuous_integration/environment-3.8-jdk11-dev.yaml + environment-file: continuous_integration/environment-3.9-jdk11-dev.yaml - name: Download the pre-build jar uses: actions/download-artifact@v1 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 693471e15..a6f862aed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,11 +147,11 @@ jobs: with: miniforge-variant: Mambaforge use-mamba: true - python-version: "3.8" + python-version: "3.9" channel-priority: strict channels: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && 'dask/label/dev,conda-forge,nodefaults' || 'conda-forge,nodefaults' }} activate-environment: dask-sql - environment-file: continuous_integration/environment-3.8-jdk11-dev.yaml + environment-file: continuous_integration/environment-3.9-jdk11-dev.yaml - name: Download the pre-build jar uses: actions/download-artifact@v1 with: From 65af34430819f2c105872162f94ee3f79ed1c0fb Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 26 May 2022 14:53:51 -0700 Subject: [PATCH 10/33] Add fugue deps to environments, bump pandas to 1.0.2 --- continuous_integration/environment-3.10-jdk11-dev.yaml | 5 +++-- continuous_integration/environment-3.10-jdk8-dev.yaml | 5 +++-- continuous_integration/environment-3.8-jdk11-dev.yaml | 5 +++-- continuous_integration/environment-3.8-jdk8-dev.yaml | 5 +++-- continuous_integration/environment-3.9-jdk11-dev.yaml | 5 +++-- continuous_integration/environment-3.9-jdk8-dev.yaml | 5 +++-- docker/conda.txt | 2 +- docs/environment.yml | 2 +- docs/requirements-docs.txt | 2 +- setup.py | 2 +- 10 files changed, 22 insertions(+), 16 deletions(-) diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index 6aa63263a..e0a991858 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - adagio -- antlr4-python3-runtime +- antlr4-python3-runtime # fugue dependency; remove when we conda install fugue - ciso8601 - dask-ml>=2022.1.22 - dask>=2022.3.0 @@ -19,7 +19,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas>=1.0.0 +- pandas>=1.0.2 - pip - pre-commit - prompt_toolkit @@ -31,6 +31,7 @@ dependencies: - pytest-xdist - pytest - python=3.10 +- qpd # fugue dependency; remove when we conda install fugue - scikit-learn>=1.0.0 - sphinx - tpot diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index 077d395c6..5992890c7 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - adagio -- antlr4-python3-runtime +- antlr4-python3-runtime # fugue dependency; remove when we conda install fugue - ciso8601 - dask-ml>=2022.1.22 - dask>=2022.3.0 @@ -19,7 +19,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas>=1.0.0 +- pandas>=1.0.2 - pip - pre-commit - prompt_toolkit @@ -31,6 +31,7 @@ dependencies: - pytest-xdist - pytest - python=3.10 +- qpd # fugue dependency; remove when we conda install fugue - scikit-learn>=1.0.0 - sphinx - tpot diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index ecd9b99da..44e7aa0c6 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - adagio -- antlr4-python3-runtime +- antlr4-python3-runtime # fugue dependency; remove when we conda install fugue - ciso8601 - dask-ml=2022.1.22 - dask=2022.3.0 @@ -19,7 +19,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas=1.0.0 +- pandas=1.0.2 - pip - pre-commit - prompt_toolkit @@ -31,6 +31,7 @@ dependencies: - pytest-xdist - pytest - python=3.8 +- qpd # fugue dependency; remove when we conda install fugue - scikit-learn=1.0.0 - sphinx - tpot diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index 6d518e3f2..f31b16a96 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - adagio -- antlr4-python3-runtime +- antlr4-python3-runtime # fugue dependency; remove when we conda install fugue - ciso8601 - dask-ml=2022.1.22 - dask=2022.3.0 @@ -19,7 +19,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas=1.0.0 +- pandas=1.0.2 - pip - pre-commit - prompt_toolkit @@ -31,6 +31,7 @@ dependencies: - pytest-xdist - pytest - python=3.8 +- qpd # fugue dependency; remove when we conda install fugue - scikit-learn=1.0.0 - sphinx - tpot diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index 83242f420..28eaa897b 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - adagio -- antlr4-python3-runtime +- antlr4-python3-runtime # fugue dependency; remove when we conda install fugue - ciso8601 - dask-ml>=2022.1.22 - dask>=2022.3.0 @@ -19,7 +19,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas>=1.0.0 +- pandas>=1.0.2 - pip - pre-commit - prompt_toolkit @@ -31,6 +31,7 @@ dependencies: - pytest-xdist - pytest - python=3.9 +- qpd # fugue dependency; remove when we conda install fugue - scikit-learn>=1.0.0 - sphinx - tpot diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index 274ebbea7..3bd0a4dd1 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - adagio -- antlr4-python3-runtime +- antlr4-python3-runtime # fugue dependency; remove when we conda install fugue - ciso8601 - dask-ml>=2022.1.22 - dask>=2022.3.0 @@ -19,7 +19,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas>=1.0.0 +- pandas>=1.0.2 - pip - pre-commit - prompt_toolkit @@ -31,6 +31,7 @@ dependencies: - pytest-xdist - pytest - python=3.9 +- qpd # fugue dependency; remove when we conda install fugue - scikit-learn>=1.0.0 - sphinx - tpot diff --git a/docker/conda.txt b/docker/conda.txt index 0382e77ca..704ed71f8 100644 --- a/docker/conda.txt +++ b/docker/conda.txt @@ -1,6 +1,6 @@ python>=3.8 dask>=2022.3.0 -pandas>=1.0.0 # below 1.0, there were no nullable ext. types +pandas>=1.0.2 # below 1.0, there were no nullable ext. types jpype1>=1.0.2 openjdk>=8 maven>=3.6.0 diff --git a/docs/environment.yml b/docs/environment.yml index ffd29270b..41b2d950b 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -9,7 +9,7 @@ dependencies: - dask-sphinx-theme>=2.0.3 - maven>=3.6.0 - dask>=2022.3.0 - - pandas>=1.0.0 + - pandas>=1.0.2 - fugue>=0.5.3 - jpype1>=1.0.2 - fastapi>=0.61.1 diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 0cc941db8..4751e6c13 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -2,7 +2,7 @@ sphinx>=4.0.0 sphinx-tabs dask-sphinx-theme>=2.0.3 dask>=2022.3.0 -pandas>=1.0.0 +pandas>=1.0.2 fugue>=0.5.3 jpype1>=1.0.2 fastapi>=0.61.1 diff --git a/setup.py b/setup.py index 6be1107dc..a327d6f94 100755 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ def build(self): setup_requires=sphinx_requirements, install_requires=[ "dask[dataframe,distributed]>=2022.3.0", - "pandas>=1.0.0", # below 1.0, there were no nullable ext. types + "pandas>=1.0.2", "jpype1>=1.0.2", "fastapi>=0.61.1", "uvicorn>=0.11.3", From b65c35038d93f7365dfb0b3111b7af112c080294 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 26 May 2022 15:17:05 -0700 Subject: [PATCH 11/33] Add back antlr4 version ceiling --- continuous_integration/environment-3.10-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.10-jdk8-dev.yaml | 2 +- continuous_integration/environment-3.8-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.8-jdk8-dev.yaml | 2 +- continuous_integration/environment-3.9-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.9-jdk8-dev.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index e0a991858..4bff38775 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - adagio -- antlr4-python3-runtime # fugue dependency; remove when we conda install fugue +- antlr4-python3-runtime<4.10 # fugue dependency; remove when we conda install fugue - ciso8601 - dask-ml>=2022.1.22 - dask>=2022.3.0 diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index 5992890c7..0a17af24a 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - adagio -- antlr4-python3-runtime # fugue dependency; remove when we conda install fugue +- antlr4-python3-runtime<4.10 # fugue dependency; remove when we conda install fugue - ciso8601 - dask-ml>=2022.1.22 - dask>=2022.3.0 diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index 44e7aa0c6..8151ffb0d 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - adagio -- antlr4-python3-runtime # fugue dependency; remove when we conda install fugue +- antlr4-python3-runtime<4.10 # fugue dependency; remove when we conda install fugue - ciso8601 - dask-ml=2022.1.22 - dask=2022.3.0 diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index f31b16a96..06ed18704 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - adagio -- antlr4-python3-runtime # fugue dependency; remove when we conda install fugue +- antlr4-python3-runtime<4.10 # fugue dependency; remove when we conda install fugue - ciso8601 - dask-ml=2022.1.22 - dask=2022.3.0 diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index 28eaa897b..554edfda9 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - adagio -- antlr4-python3-runtime # fugue dependency; remove when we conda install fugue +- antlr4-python3-runtime<4.10 # fugue dependency; remove when we conda install fugue - ciso8601 - dask-ml>=2022.1.22 - dask>=2022.3.0 diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index 3bd0a4dd1..ecc2d4457 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -4,7 +4,7 @@ channels: - nodefaults dependencies: - adagio -- antlr4-python3-runtime # fugue dependency; remove when we conda install fugue +- antlr4-python3-runtime<4.10 # fugue dependency; remove when we conda install fugue - ciso8601 - dask-ml>=2022.1.22 - dask>=2022.3.0 From a5ad5de66960e070e95eb22c94ab4508f979cecd Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 26 May 2022 15:47:49 -0700 Subject: [PATCH 12/33] Explicitly mark all fugue dependencies --- .../environment-3.10-jdk11-dev.yaml | 15 ++++++++------- .../environment-3.10-jdk8-dev.yaml | 15 ++++++++------- .../environment-3.8-jdk11-dev.yaml | 15 ++++++++------- .../environment-3.8-jdk8-dev.yaml | 15 ++++++++------- .../environment-3.9-jdk11-dev.yaml | 15 ++++++++------- .../environment-3.9-jdk8-dev.yaml | 15 ++++++++------- 6 files changed, 48 insertions(+), 42 deletions(-) diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index 4bff38775..e8e0b933e 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -3,13 +3,9 @@ channels: - conda-forge - nodefaults dependencies: -- adagio -- antlr4-python3-runtime<4.10 # fugue dependency; remove when we conda install fugue -- ciso8601 - dask-ml>=2022.1.22 - dask>=2022.3.0 - fastapi>=0.61.1 -- fs - intake>=0.6.0 - jpype1>=1.0.2 - jsonschema @@ -20,7 +16,6 @@ dependencies: - nest-asyncio - openjdk=11 - pandas>=1.0.2 -- pip - pre-commit - prompt_toolkit - psycopg2 @@ -31,12 +26,18 @@ dependencies: - pytest-xdist - pytest - python=3.10 -- qpd # fugue dependency; remove when we conda install fugue - scikit-learn>=1.0.0 - sphinx - tpot -- triad - tzlocal>=2.1 - uvicorn>=0.11.3 +# fugue dependencies; remove when we conda install fugue +- adagio +- antlr4-python3-runtime<4.10 +- ciso8601 +- fs +- pip +- qpd +- triad - pip: - fugue[sql]>=0.5.3 diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index 0a17af24a..c3159d04b 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -3,13 +3,9 @@ channels: - conda-forge - nodefaults dependencies: -- adagio -- antlr4-python3-runtime<4.10 # fugue dependency; remove when we conda install fugue -- ciso8601 - dask-ml>=2022.1.22 - dask>=2022.3.0 - fastapi>=0.61.1 -- fs - intake>=0.6.0 - jpype1>=1.0.2 - jsonschema @@ -20,7 +16,6 @@ dependencies: - nest-asyncio - openjdk=8 - pandas>=1.0.2 -- pip - pre-commit - prompt_toolkit - psycopg2 @@ -31,12 +26,18 @@ dependencies: - pytest-xdist - pytest - python=3.10 -- qpd # fugue dependency; remove when we conda install fugue - scikit-learn>=1.0.0 - sphinx - tpot -- triad - tzlocal>=2.1 - uvicorn>=0.11.3 +# fugue dependencies; remove when we conda install fugue +- adagio +- antlr4-python3-runtime<4.10 +- ciso8601 +- fs +- pip +- qpd +- triad - pip: - fugue[sql]>=0.5.3 diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index 8151ffb0d..781cfdae6 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -3,13 +3,9 @@ channels: - conda-forge - nodefaults dependencies: -- adagio -- antlr4-python3-runtime<4.10 # fugue dependency; remove when we conda install fugue -- ciso8601 - dask-ml=2022.1.22 - dask=2022.3.0 - fastapi=0.61.1 -- fs - intake=0.6.0 - jpype1=1.0.2 - jsonschema @@ -20,7 +16,6 @@ dependencies: - nest-asyncio - openjdk=11 - pandas=1.0.2 -- pip - pre-commit - prompt_toolkit - psycopg2 @@ -31,12 +26,18 @@ dependencies: - pytest-xdist - pytest - python=3.8 -- qpd # fugue dependency; remove when we conda install fugue - scikit-learn=1.0.0 - sphinx - tpot -- triad - tzlocal=2.1 - uvicorn=0.11.3 +# fugue dependencies; remove when we conda install fugue +- adagio +- antlr4-python3-runtime<4.10 +- ciso8601 +- fs +- pip +- qpd +- triad - pip: - fugue[sql]==0.5.3 diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index 06ed18704..869c4cbf3 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -3,13 +3,9 @@ channels: - conda-forge - nodefaults dependencies: -- adagio -- antlr4-python3-runtime<4.10 # fugue dependency; remove when we conda install fugue -- ciso8601 - dask-ml=2022.1.22 - dask=2022.3.0 - fastapi=0.61.1 -- fs - intake=0.6.0 - jpype1=1.0.2 - jsonschema @@ -20,7 +16,6 @@ dependencies: - nest-asyncio - openjdk=8 - pandas=1.0.2 -- pip - pre-commit - prompt_toolkit - psycopg2 @@ -31,12 +26,18 @@ dependencies: - pytest-xdist - pytest - python=3.8 -- qpd # fugue dependency; remove when we conda install fugue - scikit-learn=1.0.0 - sphinx - tpot -- triad - tzlocal=2.1 - uvicorn=0.11.3 +# fugue dependencies; remove when we conda install fugue +- adagio +- antlr4-python3-runtime<4.10 +- ciso8601 +- fs +- pip +- qpd +- triad - pip: - fugue[sql]==0.5.3 diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index 554edfda9..86d26af1b 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -3,13 +3,9 @@ channels: - conda-forge - nodefaults dependencies: -- adagio -- antlr4-python3-runtime<4.10 # fugue dependency; remove when we conda install fugue -- ciso8601 - dask-ml>=2022.1.22 - dask>=2022.3.0 - fastapi>=0.61.1 -- fs - intake>=0.6.0 - jpype1>=1.0.2 - jsonschema @@ -20,7 +16,6 @@ dependencies: - nest-asyncio - openjdk=11 - pandas>=1.0.2 -- pip - pre-commit - prompt_toolkit - psycopg2 @@ -31,12 +26,18 @@ dependencies: - pytest-xdist - pytest - python=3.9 -- qpd # fugue dependency; remove when we conda install fugue - scikit-learn>=1.0.0 - sphinx - tpot -- triad - tzlocal>=2.1 - uvicorn>=0.11.3 +# fugue dependencies; remove when we conda install fugue +- adagio +- antlr4-python3-runtime<4.10 +- ciso8601 +- fs +- pip +- qpd +- triad - pip: - fugue[sql]>=0.5.3 diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index ecc2d4457..d5c848af8 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -3,13 +3,9 @@ channels: - conda-forge - nodefaults dependencies: -- adagio -- antlr4-python3-runtime<4.10 # fugue dependency; remove when we conda install fugue -- ciso8601 - dask-ml>=2022.1.22 - dask>=2022.3.0 - fastapi>=0.61.1 -- fs - intake>=0.6.0 - jpype1>=1.0.2 - jsonschema @@ -20,7 +16,6 @@ dependencies: - nest-asyncio - openjdk=8 - pandas>=1.0.2 -- pip - pre-commit - prompt_toolkit - psycopg2 @@ -31,12 +26,18 @@ dependencies: - pytest-xdist - pytest - python=3.9 -- qpd # fugue dependency; remove when we conda install fugue - scikit-learn>=1.0.0 - sphinx - tpot -- triad - tzlocal>=2.1 - uvicorn>=0.11.3 +# fugue dependencies; remove when we conda install fugue +- adagio +- antlr4-python3-runtime<4.10 +- ciso8601 +- fs +- pip +- qpd +- triad - pip: - fugue[sql]>=0.5.3 From 6421dcf98a4d8576812e55e45ac6051e9ee61946 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 May 2022 06:29:51 -0700 Subject: [PATCH 13/33] Alter test_analyze to avoid rtol --- tests/integration/test_analyze.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_analyze.py b/tests/integration/test_analyze.py index cd51ce1b8..d3b4bd08d 100644 --- a/tests/integration/test_analyze.py +++ b/tests/integration/test_analyze.py @@ -12,11 +12,12 @@ def test_analyze(c, df): 700.0, df.a.mean(), df.a.std(), - 1.0, + df.a.min(), + # Dask's approx quantiles do not match up with pandas and must be specified explicitly + 2.0, 2.0, - 2.0, # incorrect, but what Dask gives for approx quantile - 3.0, 3.0, + df.a.max(), "double", "a", ], @@ -25,9 +26,10 @@ def test_analyze(c, df): df.b.mean(), df.b.std(), df.b.min(), - df.b.quantile(0.25), - df.b.quantile(0.5), - df.b.quantile(0.75), + # Dask's approx quantiles do not match up with pandas and must be specified explicitly + 2.73108, + 5.20286, + 7.60595, df.b.max(), "double", "b", @@ -47,9 +49,8 @@ def test_analyze(c, df): ], ) - # The percentiles are calculated only approximately, therefore we do not use exact matching - assert_eq(result_df, expected_df, rtol=0.135) + assert_eq(result_df, expected_df) result_df = c.sql("ANALYZE TABLE df COMPUTE STATISTICS FOR COLUMNS a") - assert_eq(result_df, expected_df[["a"]], rtol=0.135) + assert_eq(result_df, expected_df[["a"]]) From d9e6e97771992ef086848e481e37064ffd9bee28 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 May 2022 06:39:47 -0700 Subject: [PATCH 14/33] Bump pandas to 1.0.5 to fix upstream numpy issues --- continuous_integration/environment-3.10-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.10-jdk8-dev.yaml | 2 +- continuous_integration/environment-3.8-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.9-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.9-jdk8-dev.yaml | 2 +- docker/conda.txt | 2 +- docs/environment.yml | 2 +- docs/requirements-docs.txt | 2 +- setup.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index e8e0b933e..429f25e63 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas>=1.0.2 +- pandas>=1.0.5 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index c3159d04b..86cb21a17 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas>=1.0.2 +- pandas>=1.0.5 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index 781cfdae6..9cfb25f69 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas=1.0.2 +- pandas=1.0.5 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index 86d26af1b..8203d64ad 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas>=1.0.2 +- pandas>=1.0.5 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index d5c848af8..ee7a2f27c 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas>=1.0.2 +- pandas>=1.0.5 - pre-commit - prompt_toolkit - psycopg2 diff --git a/docker/conda.txt b/docker/conda.txt index 704ed71f8..9f109b8ac 100644 --- a/docker/conda.txt +++ b/docker/conda.txt @@ -1,6 +1,6 @@ python>=3.8 dask>=2022.3.0 -pandas>=1.0.2 # below 1.0, there were no nullable ext. types +pandas>=1.0.5 jpype1>=1.0.2 openjdk>=8 maven>=3.6.0 diff --git a/docs/environment.yml b/docs/environment.yml index 41b2d950b..c3bcb3024 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -9,7 +9,7 @@ dependencies: - dask-sphinx-theme>=2.0.3 - maven>=3.6.0 - dask>=2022.3.0 - - pandas>=1.0.2 + - pandas>=1.0.5 - fugue>=0.5.3 - jpype1>=1.0.2 - fastapi>=0.61.1 diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 4751e6c13..b34b1e946 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -2,7 +2,7 @@ sphinx>=4.0.0 sphinx-tabs dask-sphinx-theme>=2.0.3 dask>=2022.3.0 -pandas>=1.0.2 +pandas>=1.0.5 fugue>=0.5.3 jpype1>=1.0.2 fastapi>=0.61.1 diff --git a/setup.py b/setup.py index a327d6f94..058efd929 100755 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ def build(self): setup_requires=sphinx_requirements, install_requires=[ "dask[dataframe,distributed]>=2022.3.0", - "pandas>=1.0.2", + "pandas>=1.0.5", "jpype1>=1.0.2", "fastapi>=0.61.1", "uvicorn>=0.11.3", From 34383e8b15f175e538a40219e0e799ad99928e8e Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 May 2022 06:41:40 -0700 Subject: [PATCH 15/33] Alter datetime casting util to dodge panda casting failures --- tests/integration/test_compatibility.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_compatibility.py b/tests/integration/test_compatibility.py index 63c1668b2..ec2949229 100644 --- a/tests/integration/test_compatibility.py +++ b/tests/integration/test_compatibility.py @@ -22,8 +22,11 @@ def cast_datetime_to_string(df): cols = df.select_dtypes(include=["datetime64[ns]"]).columns.tolist() - # Casting to object first as - # directly converting to string looses second precision + + if not cols: + return df + + # Casting directly to string loses second precision df[cols] = df[cols].astype("object").astype("string") return df From 62c16efb410093b83c0b058fdac270267e8de513 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 May 2022 06:55:14 -0700 Subject: [PATCH 16/33] Bump pandas to 1.1.0 for groupby dropna support --- continuous_integration/environment-3.10-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.10-jdk8-dev.yaml | 2 +- continuous_integration/environment-3.8-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.9-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.9-jdk8-dev.yaml | 2 +- docker/conda.txt | 2 +- docs/environment.yml | 2 +- docs/requirements-docs.txt | 2 +- setup.py | 2 +- tests/integration/test_groupby.py | 3 --- 10 files changed, 9 insertions(+), 12 deletions(-) diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index 429f25e63..2e017e422 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas>=1.0.5 +- pandas>=1.1.0 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index 86cb21a17..e544d02f5 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas>=1.0.5 +- pandas>=1.1.0 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index 9cfb25f69..9a666a8b6 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas=1.0.5 +- pandas=1.1.0 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index 8203d64ad..8908c30ea 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas>=1.0.5 +- pandas>=1.1.0 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index ee7a2f27c..892485176 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas>=1.0.5 +- pandas>=1.1.0 - pre-commit - prompt_toolkit - psycopg2 diff --git a/docker/conda.txt b/docker/conda.txt index 9f109b8ac..72e9aaeaa 100644 --- a/docker/conda.txt +++ b/docker/conda.txt @@ -1,6 +1,6 @@ python>=3.8 dask>=2022.3.0 -pandas>=1.0.5 +pandas>=1.1.0 jpype1>=1.0.2 openjdk>=8 maven>=3.6.0 diff --git a/docs/environment.yml b/docs/environment.yml index c3bcb3024..ca9ea9b9e 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -9,7 +9,7 @@ dependencies: - dask-sphinx-theme>=2.0.3 - maven>=3.6.0 - dask>=2022.3.0 - - pandas>=1.0.5 + - pandas>=1.1.0 - fugue>=0.5.3 - jpype1>=1.0.2 - fastapi>=0.61.1 diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index b34b1e946..f91839691 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -2,7 +2,7 @@ sphinx>=4.0.0 sphinx-tabs dask-sphinx-theme>=2.0.3 dask>=2022.3.0 -pandas>=1.0.5 +pandas>=1.1.0 fugue>=0.5.3 jpype1>=1.0.2 fastapi>=0.61.1 diff --git a/setup.py b/setup.py index 058efd929..c89d0aa1e 100755 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ def build(self): setup_requires=sphinx_requirements, install_requires=[ "dask[dataframe,distributed]>=2022.3.0", - "pandas>=1.0.5", + "pandas>=1.1.0", "jpype1>=1.0.2", "fastapi>=0.61.1", "uvicorn>=0.11.3", diff --git a/tests/integration/test_groupby.py b/tests/integration/test_groupby.py index 658ad4fa2..0ef7ab9c5 100644 --- a/tests/integration/test_groupby.py +++ b/tests/integration/test_groupby.py @@ -152,12 +152,9 @@ def test_group_by_nan(c): ) expected_df = pd.DataFrame({"c": [3, float("nan"), 1]}) - # The dtype in pandas 1.0.5 and pandas 1.1.0 are different, so - # we cannot check here assert_eq( return_df.sort_values("c").reset_index(drop=True), expected_df.sort_values("c").reset_index(drop=True), - check_dtype=False, ) return_df = c.sql( From 6b7358da89f771adff268e04ef9a29286efbf8a7 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 May 2022 08:27:36 -0700 Subject: [PATCH 17/33] Simplify string dtype check for get_supported_aggregations --- dask_sql/physical/rel/logical/aggregate.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dask_sql/physical/rel/logical/aggregate.py b/dask_sql/physical/rel/logical/aggregate.py index 0dc3bc926..5cad4476c 100644 --- a/dask_sql/physical/rel/logical/aggregate.py +++ b/dask_sql/physical/rel/logical/aggregate.py @@ -81,11 +81,8 @@ def get_supported_aggregation(self, series): if "cudf" in str(series._partition_type): return built_in_aggregation - # With pandas StringDtype built-in aggregations work - # while with pandas ObjectDtype and Nulls built-in aggregations fail - if isinstance(series, dd.Series) and isinstance( - series.dtype, pd.StringDtype - ): + # with pandas StringDtype built-in aggregations work + if isinstance(series.dtype, pd.StringDtype): return built_in_aggregation return self.custom_aggregation @@ -375,6 +372,7 @@ def _perform_aggregation( # format aggregations for Dask; also check if we can use fast path for # groupby, which is only supported if we are not using any custom aggregations + # and our pandas version support dropna for groupbys aggregations_dict = defaultdict(dict) fast_groupby = True for aggregation in aggregations: From df28ea88d003dc45d1cb8c3f4b2f2334f7a409f6 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 May 2022 08:28:11 -0700 Subject: [PATCH 18/33] Add check_dtype=False back to test_group_by_nan --- tests/integration/test_groupby.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_groupby.py b/tests/integration/test_groupby.py index 0ef7ab9c5..309d25b60 100644 --- a/tests/integration/test_groupby.py +++ b/tests/integration/test_groupby.py @@ -152,10 +152,8 @@ def test_group_by_nan(c): ) expected_df = pd.DataFrame({"c": [3, float("nan"), 1]}) - assert_eq( - return_df.sort_values("c").reset_index(drop=True), - expected_df.sort_values("c").reset_index(drop=True), - ) + # we return nullable int dtype instead of float + assert_eq(return_df, expected_df, check_dtype=False) return_df = c.sql( """ From c7ef83555dfca490b8d4707a07a0eb8449f0d48a Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 May 2022 09:20:41 -0700 Subject: [PATCH 19/33] Bump cluster to python 3.9 --- .github/cluster.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/cluster.yml b/.github/cluster.yml index 24f75a35b..dc75c064c 100644 --- a/.github/cluster.yml +++ b/.github/cluster.yml @@ -3,13 +3,13 @@ version: '3' services: dask-scheduler: container_name: dask-scheduler - image: daskdev/dask:dev + image: daskdev/dask:dev-py3.9 command: dask-scheduler ports: - "8786:8786" dask-worker: container_name: dask-worker - image: daskdev/dask:dev + image: daskdev/dask:dev-py3.9 command: dask-worker dask-scheduler:8786 environment: USE_MAMBA: "true" From d254e1f0758e78c904f74d6462344514799ae1ca Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 May 2022 10:09:31 -0700 Subject: [PATCH 20/33] Bump fastapi to 0.69.0, resolve remaining JDBC failures --- .../environment-3.10-jdk11-dev.yaml | 2 +- .../environment-3.10-jdk8-dev.yaml | 2 +- .../environment-3.8-jdk11-dev.yaml | 2 +- .../environment-3.8-jdk8-dev.yaml | 2 +- .../environment-3.9-jdk11-dev.yaml | 2 +- .../environment-3.9-jdk8-dev.yaml | 2 +- continuous_integration/recipe/meta.yaml | 2 +- docker/conda.txt | 2 +- docker/main.dockerfile | 2 +- docs/environment.yml | 2 +- docs/requirements-docs.txt | 2 +- setup.py | 2 +- tests/integration/test_jdbc.py | 92 ++----------------- 13 files changed, 22 insertions(+), 94 deletions(-) diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index 2e017e422..4529332f0 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -5,7 +5,7 @@ channels: dependencies: - dask-ml>=2022.1.22 - dask>=2022.3.0 -- fastapi>=0.61.1 +- fastapi>=0.69.0 - intake>=0.6.0 - jpype1>=1.0.2 - jsonschema diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index e544d02f5..563e1215d 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -5,7 +5,7 @@ channels: dependencies: - dask-ml>=2022.1.22 - dask>=2022.3.0 -- fastapi>=0.61.1 +- fastapi>=0.69.0 - intake>=0.6.0 - jpype1>=1.0.2 - jsonschema diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index 9a666a8b6..747f7b8ed 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -5,7 +5,7 @@ channels: dependencies: - dask-ml=2022.1.22 - dask=2022.3.0 -- fastapi=0.61.1 +- fastapi=0.69.0 - intake=0.6.0 - jpype1=1.0.2 - jsonschema diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index 869c4cbf3..3639ebae3 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -5,7 +5,7 @@ channels: dependencies: - dask-ml=2022.1.22 - dask=2022.3.0 -- fastapi=0.61.1 +- fastapi=0.69.0 - intake=0.6.0 - jpype1=1.0.2 - jsonschema diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index 8908c30ea..1f5832839 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -5,7 +5,7 @@ channels: dependencies: - dask-ml>=2022.1.22 - dask>=2022.3.0 -- fastapi>=0.61.1 +- fastapi>=0.69.0 - intake>=0.6.0 - jpype1>=1.0.2 - jsonschema diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index 892485176..f445e81fb 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -5,7 +5,7 @@ channels: dependencies: - dask-ml>=2022.1.22 - dask>=2022.3.0 -- fastapi>=0.61.1 +- fastapi>=0.69.0 - intake>=0.6.0 - jpype1>=1.0.2 - jsonschema diff --git a/continuous_integration/recipe/meta.yaml b/continuous_integration/recipe/meta.yaml index b2a40190a..65512ca47 100644 --- a/continuous_integration/recipe/meta.yaml +++ b/continuous_integration/recipe/meta.yaml @@ -32,7 +32,7 @@ requirements: - pandas >=1.0.0 - jpype1 >=1.0.2 - openjdk >=8 - - fastapi >=0.61.1 + - fastapi >=0.69.0 - uvicorn >=0.11.3 - tzlocal >=2.1 - prompt-toolkit diff --git a/docker/conda.txt b/docker/conda.txt index 72e9aaeaa..8959db442 100644 --- a/docker/conda.txt +++ b/docker/conda.txt @@ -10,7 +10,7 @@ pytest-xdist mock>=4.0.3 sphinx>=3.2.1 tzlocal>=2.1 -fastapi>=0.61.1 +fastapi>=0.69.0 nest-asyncio>=1.4.3 uvicorn>=0.11.3 pyarrow>=1.0.1 diff --git a/docker/main.dockerfile b/docker/main.dockerfile index 297ad710b..21eaf6df2 100644 --- a/docker/main.dockerfile +++ b/docker/main.dockerfile @@ -11,7 +11,7 @@ RUN conda config --add channels conda-forge \ "openjdk>=11" \ "maven>=3.6.0" \ "tzlocal>=2.1" \ - "fastapi>=0.61.1" \ + "fastapi>=0.69.0" \ "uvicorn>=0.11.3" \ "pyarrow>=1.0.1" \ "prompt_toolkit>=3.0.8" \ diff --git a/docs/environment.yml b/docs/environment.yml index ca9ea9b9e..865c75d3a 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -12,7 +12,7 @@ dependencies: - pandas>=1.1.0 - fugue>=0.5.3 - jpype1>=1.0.2 - - fastapi>=0.61.1 + - fastapi>=0.69.0 - uvicorn>=0.11.3 - tzlocal>=2.1 - prompt_toolkit diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index f91839691..04a053ae1 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -5,7 +5,7 @@ dask>=2022.3.0 pandas>=1.1.0 fugue>=0.5.3 jpype1>=1.0.2 -fastapi>=0.61.1 +fastapi>=0.69.0 uvicorn>=0.11.3 tzlocal>=2.1 prompt_toolkit diff --git a/setup.py b/setup.py index c89d0aa1e..6b333dc02 100755 --- a/setup.py +++ b/setup.py @@ -90,7 +90,7 @@ def build(self): "dask[dataframe,distributed]>=2022.3.0", "pandas>=1.1.0", "jpype1>=1.0.2", - "fastapi>=0.61.1", + "fastapi>=0.69.0", "uvicorn>=0.11.3", "tzlocal>=2.1", "prompt_toolkit", diff --git a/tests/integration/test_jdbc.py b/tests/integration/test_jdbc.py index ce216da84..62ce7d772 100644 --- a/tests/integration/test_jdbc.py +++ b/tests/integration/test_jdbc.py @@ -100,89 +100,17 @@ def test_jdbc_has_columns(app_client, c): data=f"SELECT * from system.jdbc.columns where TABLE_NAME = '{table}'", ) assert response.status_code == 200 - result = get_result_or_error(app_client, response) + client_result = get_result_or_error(app_client, response) - assert_result(result, 24, 3) - assert result["data"] == [ - [ - "", - "a_schema", - "a_table", - "A_STR", - "VARCHAR", - "VARCHAR", - "", - "", - "", - "", - "", - "", - "", - "VARCHAR", - "", - "", - "1", - "", - "", - "", - "", - "", - "", - "", - ], - [ - "", - "a_schema", - "a_table", - "AN_INT", - "INTEGER", - "INTEGER", - "", - "", - "", - "", - "", - "", - "", - "INTEGER", - "", - "", - "2", - "", - "", - "", - "", - "", - "", - "", - ], - [ - "", - "a_schema", - "a_table", - "A_FLOAT", - "FLOAT", - "FLOAT", - "", - "", - "", - "", - "", - "", - "", - "FLOAT", - "", - "", - "3", - "", - "", - "", - "", - "", - "", - "", - ], - ] + # ordering of rows isn't consistent between fastapi versions + context_result = ( + c.sql("SELECT * FROM system_jdbc.columns WHERE TABLE_NAME = 'a_table'") + .compute() + .values.tolist() + ) + + assert_result(client_result, 24, 3) + assert client_result["data"] == context_result def assert_result(result, col_len, data_len): From 67b0ff3777e5f9660d43aa5193d8db873a3fba31 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 May 2022 10:42:49 -0700 Subject: [PATCH 21/33] Typo - correct pandas version --- continuous_integration/environment-3.8-jdk8-dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index 3639ebae3..bd7445e8f 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas=1.0.2 +- pandas=1.1.0 - pre-commit - prompt_toolkit - psycopg2 From 8bec354f2fd33ee67ad9320ffe4c172af73fd9df Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 May 2022 13:37:30 -0700 Subject: [PATCH 22/33] Generalize test_multi_case_when's dtype check --- tests/integration/test_select.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_select.py b/tests/integration/test_select.py index 6f93692f9..f5c4b7911 100644 --- a/tests/integration/test_select.py +++ b/tests/integration/test_select.py @@ -204,6 +204,7 @@ def test_multi_case_when(c): FROM df """ ) - expected_df = pd.DataFrame({"C": [0, 1, 1, 1, 0]}, dtype=np.int32) + expected_df = pd.DataFrame({"C": [0, 1, 1, 1, 0]}) - assert_eq(actual_df, expected_df) + # dtype varies between int32/int64 depending on pandas version + assert_eq(actual_df, expected_df, check_dtype=False) From 203487b7b99849db52e6af9a27fbf7114a0b719f Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 1 Jun 2022 06:59:22 -0700 Subject: [PATCH 23/33] Bump pandas to 1.1.1 to resolve flaky test failures --- continuous_integration/environment-3.10-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.10-jdk8-dev.yaml | 2 +- continuous_integration/environment-3.8-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.8-jdk8-dev.yaml | 2 +- continuous_integration/environment-3.9-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.9-jdk8-dev.yaml | 2 +- docker/conda.txt | 2 +- docs/environment.yml | 2 +- docs/requirements-docs.txt | 2 +- setup.py | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index 4529332f0..e971aba5d 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas>=1.1.0 +- pandas>=1.1.1 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index 563e1215d..352405ec1 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas>=1.1.0 +- pandas>=1.1.1 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index 747f7b8ed..797f58464 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas=1.1.0 +- pandas=1.1.1 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index bd7445e8f..de81968b7 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas=1.1.0 +- pandas=1.1.1 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index 1f5832839..d50f4ff52 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas>=1.1.0 +- pandas>=1.1.1 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index f445e81fb..8cc1421ac 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas>=1.1.0 +- pandas>=1.1.1 - pre-commit - prompt_toolkit - psycopg2 diff --git a/docker/conda.txt b/docker/conda.txt index 8959db442..10d6e2e46 100644 --- a/docker/conda.txt +++ b/docker/conda.txt @@ -1,6 +1,6 @@ python>=3.8 dask>=2022.3.0 -pandas>=1.1.0 +pandas>=1.1.1 jpype1>=1.0.2 openjdk>=8 maven>=3.6.0 diff --git a/docs/environment.yml b/docs/environment.yml index 865c75d3a..b7de026e2 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -9,7 +9,7 @@ dependencies: - dask-sphinx-theme>=2.0.3 - maven>=3.6.0 - dask>=2022.3.0 - - pandas>=1.1.0 + - pandas>=1.1.1 - fugue>=0.5.3 - jpype1>=1.0.2 - fastapi>=0.69.0 diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 04a053ae1..4bce6f30a 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -2,7 +2,7 @@ sphinx>=4.0.0 sphinx-tabs dask-sphinx-theme>=2.0.3 dask>=2022.3.0 -pandas>=1.1.0 +pandas>=1.1.1 fugue>=0.5.3 jpype1>=1.0.2 fastapi>=0.69.0 diff --git a/setup.py b/setup.py index 6b333dc02..46ee8375b 100755 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ def build(self): setup_requires=sphinx_requirements, install_requires=[ "dask[dataframe,distributed]>=2022.3.0", - "pandas>=1.1.0", + "pandas>=1.1.1", "jpype1>=1.0.2", "fastapi>=0.69.0", "uvicorn>=0.11.3", From 70821e59d32f77e5e6d14ec08c3698d71443ea13 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 1 Jun 2022 09:40:48 -0700 Subject: [PATCH 24/33] Constrain mlflow for windows python 3.8 testing --- continuous_integration/environment-3.8-jdk11-dev.yaml | 2 ++ continuous_integration/environment-3.8-jdk8-dev.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index 797f58464..f336e2cac 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -12,6 +12,8 @@ dependencies: - lightgbm - maven - mlflow +# remove windows constraint when mlflow issues are resolved +- mlflow<1.26.1 # [win] - mock - nest-asyncio - openjdk=11 diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index de81968b7..e5b7c6d19 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -12,6 +12,8 @@ dependencies: - lightgbm - maven - mlflow +# remove windows constraint when mlflow issues are resolved +- mlflow<1.26.1 # [win] - mock - nest-asyncio - openjdk=8 From 23cdf5e23dc5656bcbcc9d57b8c0d3075922da14 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 1 Jun 2022 09:50:19 -0700 Subject: [PATCH 25/33] Selectors don't work for conda env files --- continuous_integration/environment-3.8-jdk11-dev.yaml | 4 +--- continuous_integration/environment-3.8-jdk8-dev.yaml | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index f336e2cac..a969d9573 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -11,9 +11,7 @@ dependencies: - jsonschema - lightgbm - maven -- mlflow -# remove windows constraint when mlflow issues are resolved -- mlflow<1.26.1 # [win] +- mlflow<1.26.1 # remove constraint when Windows mlflow issues are resolved - mock - nest-asyncio - openjdk=11 diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index e5b7c6d19..b2e8f45ec 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -11,9 +11,7 @@ dependencies: - jsonschema - lightgbm - maven -- mlflow -# remove windows constraint when mlflow issues are resolved -- mlflow<1.26.1 # [win] +- mlflow<1.26.1 # remove constraint when Windows mlflow issues are resolved - mock - nest-asyncio - openjdk=8 From 5e04d345a86095176f72b2799ca4657d4f75956a Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 1 Jun 2022 11:06:24 -0700 Subject: [PATCH 26/33] Problems seem to persist in 1.1.1, bump to 1.1.2 --- .../environment-3.10-jdk11-dev.yaml | 2 +- .../environment-3.10-jdk8-dev.yaml | 2 +- .../environment-3.8-jdk11-dev.yaml | 2 +- .../environment-3.8-jdk8-dev.yaml | 2 +- .../environment-3.9-jdk11-dev.yaml | 2 +- .../environment-3.9-jdk8-dev.yaml | 2 +- docker/conda.txt | 2 +- docs/environment.yml | 2 +- docs/requirements-docs.txt | 2 +- setup.py | 2 +- tests/integration/fixtures.py | 3 + tests/integration/test_postgres.py | 60 +++++++++---------- 12 files changed, 43 insertions(+), 40 deletions(-) diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index e971aba5d..936a78d21 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas>=1.1.1 +- pandas>=1.1.2 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index 352405ec1..915ba3e10 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas>=1.1.1 +- pandas>=1.1.2 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index a969d9573..e0d3f6a97 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas=1.1.1 +- pandas=1.1.2 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index b2e8f45ec..5e7290ba9 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas=1.1.1 +- pandas=1.1.2 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index d50f4ff52..2a22466a0 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=11 -- pandas>=1.1.1 +- pandas>=1.1.2 - pre-commit - prompt_toolkit - psycopg2 diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index 8cc1421ac..5e0d67541 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -15,7 +15,7 @@ dependencies: - mock - nest-asyncio - openjdk=8 -- pandas>=1.1.1 +- pandas>=1.1.2 - pre-commit - prompt_toolkit - psycopg2 diff --git a/docker/conda.txt b/docker/conda.txt index 10d6e2e46..183597c22 100644 --- a/docker/conda.txt +++ b/docker/conda.txt @@ -1,6 +1,6 @@ python>=3.8 dask>=2022.3.0 -pandas>=1.1.1 +pandas>=1.1.2 jpype1>=1.0.2 openjdk>=8 maven>=3.6.0 diff --git a/docs/environment.yml b/docs/environment.yml index b7de026e2..f0b4eafac 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -9,7 +9,7 @@ dependencies: - dask-sphinx-theme>=2.0.3 - maven>=3.6.0 - dask>=2022.3.0 - - pandas>=1.1.1 + - pandas>=1.1.2 - fugue>=0.5.3 - jpype1>=1.0.2 - fastapi>=0.69.0 diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 4bce6f30a..b370b9ed6 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -2,7 +2,7 @@ sphinx>=4.0.0 sphinx-tabs dask-sphinx-theme>=2.0.3 dask>=2022.3.0 -pandas>=1.1.1 +pandas>=1.1.2 fugue>=0.5.3 jpype1>=1.0.2 fastapi>=0.69.0 diff --git a/setup.py b/setup.py index 46ee8375b..d1ff97527 100755 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ def build(self): setup_requires=sphinx_requirements, install_requires=[ "dask[dataframe,distributed]>=2022.3.0", - "pandas>=1.1.1", + "pandas>=1.1.2", "jpype1>=1.0.2", "fastapi>=0.69.0", "uvicorn>=0.11.3", diff --git a/tests/integration/fixtures.py b/tests/integration/fixtures.py index e1f53c6db..4f73fb679 100644 --- a/tests/integration/fixtures.py +++ b/tests/integration/fixtures.py @@ -287,6 +287,9 @@ def assert_query_gives_same_result(engine): def _assert_query_gives_same_result(query, sort_columns=None, **kwargs): sql_result = pd.read_sql_query(query, engine) dask_result = c.sql(query).compute() + res = c.sql(query) + + breakpoint() # allow that the names are different # as expressions are handled differently diff --git a/tests/integration/test_postgres.py b/tests/integration/test_postgres.py index f852dadac..80328f27d 100644 --- a/tests/integration/test_postgres.py +++ b/tests/integration/test_postgres.py @@ -260,29 +260,29 @@ def test_string_operations(assert_query_gives_same_result): def test_statistical_functions(assert_query_gives_same_result): # test regr_count - assert_query_gives_same_result( - """ - select user_id, REGR_COUNT(a,b) FROM df1 GROUP BY user_id - """, - ["user_id"], - check_names=False, - ) - - assert_query_gives_same_result( - """ - select user_id, REGR_SXX(a, 1.0 * b) FROM df1 GROUP BY user_id - """, - ["user_id"], - check_names=False, - ) - - assert_query_gives_same_result( - """ - select user_id, REGR_SYY(a, 1.0 * b) FROM df1 GROUP BY user_id - """, - ["user_id"], - check_names=False, - ) + # assert_query_gives_same_result( + # """ + # select user_id, REGR_COUNT(a,b) FROM df1 GROUP BY user_id + # """, + # ["user_id"], + # check_names=False, + # ) + + # assert_query_gives_same_result( + # """ + # select user_id, REGR_SXX(a, 1.0 * b) FROM df1 GROUP BY user_id + # """, + # ["user_id"], + # check_names=False, + # ) + + # assert_query_gives_same_result( + # """ + # select user_id, REGR_SYY(a, 1.0 * b) FROM df1 GROUP BY user_id + # """, + # ["user_id"], + # check_names=False, + # ) assert_query_gives_same_result( """ @@ -292,10 +292,10 @@ def test_statistical_functions(assert_query_gives_same_result): check_names=False, ) - assert_query_gives_same_result( - """ - select user_id,COVAR_SAMP(a,b) FROM df1 GROUP BY user_id - """, - ["user_id"], - check_names=False, - ) + # assert_query_gives_same_result( + # """ + # select user_id,COVAR_SAMP(a,b) FROM df1 GROUP BY user_id + # """, + # ["user_id"], + # check_names=False, + # ) From bd7cd029ce236a736b739b55a65bb5e8512beadc Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 1 Jun 2022 11:52:16 -0700 Subject: [PATCH 27/33] Remove accidental debug changes --- tests/integration/fixtures.py | 3 -- tests/integration/test_postgres.py | 60 +++++++++++++++--------------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/tests/integration/fixtures.py b/tests/integration/fixtures.py index 4f73fb679..e1f53c6db 100644 --- a/tests/integration/fixtures.py +++ b/tests/integration/fixtures.py @@ -287,9 +287,6 @@ def assert_query_gives_same_result(engine): def _assert_query_gives_same_result(query, sort_columns=None, **kwargs): sql_result = pd.read_sql_query(query, engine) dask_result = c.sql(query).compute() - res = c.sql(query) - - breakpoint() # allow that the names are different # as expressions are handled differently diff --git a/tests/integration/test_postgres.py b/tests/integration/test_postgres.py index 80328f27d..f852dadac 100644 --- a/tests/integration/test_postgres.py +++ b/tests/integration/test_postgres.py @@ -260,29 +260,29 @@ def test_string_operations(assert_query_gives_same_result): def test_statistical_functions(assert_query_gives_same_result): # test regr_count - # assert_query_gives_same_result( - # """ - # select user_id, REGR_COUNT(a,b) FROM df1 GROUP BY user_id - # """, - # ["user_id"], - # check_names=False, - # ) - - # assert_query_gives_same_result( - # """ - # select user_id, REGR_SXX(a, 1.0 * b) FROM df1 GROUP BY user_id - # """, - # ["user_id"], - # check_names=False, - # ) - - # assert_query_gives_same_result( - # """ - # select user_id, REGR_SYY(a, 1.0 * b) FROM df1 GROUP BY user_id - # """, - # ["user_id"], - # check_names=False, - # ) + assert_query_gives_same_result( + """ + select user_id, REGR_COUNT(a,b) FROM df1 GROUP BY user_id + """, + ["user_id"], + check_names=False, + ) + + assert_query_gives_same_result( + """ + select user_id, REGR_SXX(a, 1.0 * b) FROM df1 GROUP BY user_id + """, + ["user_id"], + check_names=False, + ) + + assert_query_gives_same_result( + """ + select user_id, REGR_SYY(a, 1.0 * b) FROM df1 GROUP BY user_id + """, + ["user_id"], + check_names=False, + ) assert_query_gives_same_result( """ @@ -292,10 +292,10 @@ def test_statistical_functions(assert_query_gives_same_result): check_names=False, ) - # assert_query_gives_same_result( - # """ - # select user_id,COVAR_SAMP(a,b) FROM df1 GROUP BY user_id - # """, - # ["user_id"], - # check_names=False, - # ) + assert_query_gives_same_result( + """ + select user_id,COVAR_SAMP(a,b) FROM df1 GROUP BY user_id + """, + ["user_id"], + check_names=False, + ) From 92a240fa4ad3e2a215bf795260f4f9bb1cebbac2 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 1 Jun 2022 12:28:30 -0700 Subject: [PATCH 28/33] [test-upstream] From 4ca56e87638d04d4b081577962a56b359ba88187 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 1 Jun 2022 12:56:41 -0700 Subject: [PATCH 29/33] Use python 3.9 for upstream cluster testing [test-upstream] --- .github/cluster-upstream.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/cluster-upstream.yml b/.github/cluster-upstream.yml index e74fa9686..2b0ab2070 100644 --- a/.github/cluster-upstream.yml +++ b/.github/cluster-upstream.yml @@ -3,13 +3,13 @@ version: '3' services: dask-scheduler: container_name: dask-scheduler - image: daskdev/dask:dev + image: daskdev/dask:dev-py3.9 command: dask-scheduler ports: - "8786:8786" dask-worker: container_name: dask-worker - image: daskdev/dask:dev + image: daskdev/dask:dev-py3.9 command: dask-worker dask-scheduler:8786 environment: USE_MAMBA: "true" From 46f8074346f2f33388536a17a80d463c71d52c68 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 15 Jun 2022 07:05:18 -0700 Subject: [PATCH 30/33] Updated missed pandas pinning --- continuous_integration/recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/recipe/meta.yaml b/continuous_integration/recipe/meta.yaml index 65512ca47..cd5abd580 100644 --- a/continuous_integration/recipe/meta.yaml +++ b/continuous_integration/recipe/meta.yaml @@ -29,7 +29,7 @@ requirements: run: - python - dask >=2022.3.0 - - pandas >=1.0.0 + - pandas >=1.1.2 - jpype1 >=1.0.2 - openjdk >=8 - fastapi >=0.69.0 From e8b020864d83b4b0d50ca66f730f1867a229c06f Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 15 Jun 2022 07:20:30 -0700 Subject: [PATCH 31/33] Unconstrain mlflow to see if Windows failures persist --- continuous_integration/environment-3.8-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.8-jdk8-dev.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index e0d3f6a97..0c27cc40a 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -11,7 +11,7 @@ dependencies: - jsonschema - lightgbm - maven -- mlflow<1.26.1 # remove constraint when Windows mlflow issues are resolved +- mlflow - mock - nest-asyncio - openjdk=11 diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index 5e7290ba9..338b61f3a 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -11,7 +11,7 @@ dependencies: - jsonschema - lightgbm - maven -- mlflow<1.26.1 # remove constraint when Windows mlflow issues are resolved +- mlflow - mock - nest-asyncio - openjdk=8 From 65c4196e6cdf06f3fb5ca453ee00de43f122eff9 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 15 Jun 2022 07:56:01 -0700 Subject: [PATCH 32/33] Add min version for protobuf --- continuous_integration/environment-3.8-jdk11-dev.yaml | 1 + continuous_integration/environment-3.8-jdk8-dev.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index 0c27cc40a..adddaa017 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -18,6 +18,7 @@ dependencies: - pandas=1.1.2 - pre-commit - prompt_toolkit +- protobuf>=3.20.1 - psycopg2 - pyarrow=1.0.1 - pygments diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index 338b61f3a..8d0cd68ea 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -18,6 +18,7 @@ dependencies: - pandas=1.1.2 - pre-commit - prompt_toolkit +- protobuf>=3.20.1 - psycopg2 - pyarrow=1.0.1 - pygments From 80b2223d1c11e2e74df6e430e03938b8ab5464db Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 15 Jun 2022 08:09:53 -0700 Subject: [PATCH 33/33] Bump pyarrow min version to allow for newer protobuf versions --- .github/cluster-upstream.yml | 2 +- .github/cluster.yml | 2 +- continuous_integration/environment-3.10-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.10-jdk8-dev.yaml | 2 +- continuous_integration/environment-3.8-jdk11-dev.yaml | 3 +-- continuous_integration/environment-3.8-jdk8-dev.yaml | 3 +-- continuous_integration/environment-3.9-jdk11-dev.yaml | 2 +- continuous_integration/environment-3.9-jdk8-dev.yaml | 2 +- docker/conda.txt | 2 +- docker/main.dockerfile | 2 +- setup.py | 2 +- 11 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/cluster-upstream.yml b/.github/cluster-upstream.yml index 602684f05..3bebbde0c 100644 --- a/.github/cluster-upstream.yml +++ b/.github/cluster-upstream.yml @@ -16,6 +16,6 @@ services: command: dask-worker dask-scheduler:8786 environment: USE_MAMBA: "true" - EXTRA_CONDA_PACKAGES: "dask/label/dev::dask cloudpickle>=2.1.0 pyarrow>=1.0.1 libstdcxx-ng>=12.1.0" + EXTRA_CONDA_PACKAGES: "dask/label/dev::dask cloudpickle>=2.1.0 pyarrow>=3.0.0 libstdcxx-ng>=12.1.0" volumes: - /tmp:/tmp diff --git a/.github/cluster.yml b/.github/cluster.yml index dc75c064c..da9530ee7 100644 --- a/.github/cluster.yml +++ b/.github/cluster.yml @@ -13,6 +13,6 @@ services: command: dask-worker dask-scheduler:8786 environment: USE_MAMBA: "true" - EXTRA_CONDA_PACKAGES: "cloudpickle>=2.1.0 pyarrow>=1.0.1 libstdcxx-ng>=12.1.0" + EXTRA_CONDA_PACKAGES: "cloudpickle>=2.1.0 pyarrow>=3.0.0 libstdcxx-ng>=12.1.0" volumes: - /tmp:/tmp diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-jdk11-dev.yaml index 936a78d21..8af02bb32 100644 --- a/continuous_integration/environment-3.10-jdk11-dev.yaml +++ b/continuous_integration/environment-3.10-jdk11-dev.yaml @@ -19,7 +19,7 @@ dependencies: - pre-commit - prompt_toolkit - psycopg2 -- pyarrow>=1.0.1 +- pyarrow>=3.0.0 - pygments - pyhive - pytest-cov diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml index 915ba3e10..ed83ed51a 100644 --- a/continuous_integration/environment-3.10-jdk8-dev.yaml +++ b/continuous_integration/environment-3.10-jdk8-dev.yaml @@ -19,7 +19,7 @@ dependencies: - pre-commit - prompt_toolkit - psycopg2 -- pyarrow>=1.0.1 +- pyarrow>=3.0.0 - pygments - pyhive - pytest-cov diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-jdk11-dev.yaml index adddaa017..b865bb82f 100644 --- a/continuous_integration/environment-3.8-jdk11-dev.yaml +++ b/continuous_integration/environment-3.8-jdk11-dev.yaml @@ -18,9 +18,8 @@ dependencies: - pandas=1.1.2 - pre-commit - prompt_toolkit -- protobuf>=3.20.1 - psycopg2 -- pyarrow=1.0.1 +- pyarrow=3.0.0 - pygments - pyhive - pytest-cov diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml index 8d0cd68ea..b143fcc6c 100644 --- a/continuous_integration/environment-3.8-jdk8-dev.yaml +++ b/continuous_integration/environment-3.8-jdk8-dev.yaml @@ -18,9 +18,8 @@ dependencies: - pandas=1.1.2 - pre-commit - prompt_toolkit -- protobuf>=3.20.1 - psycopg2 -- pyarrow=1.0.1 +- pyarrow=3.0.0 - pygments - pyhive - pytest-cov diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-jdk11-dev.yaml index 2a22466a0..1e4430cae 100644 --- a/continuous_integration/environment-3.9-jdk11-dev.yaml +++ b/continuous_integration/environment-3.9-jdk11-dev.yaml @@ -19,7 +19,7 @@ dependencies: - pre-commit - prompt_toolkit - psycopg2 -- pyarrow>=1.0.1 +- pyarrow>=3.0.0 - pygments - pyhive - pytest-cov diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml index 5e0d67541..314aaadd0 100644 --- a/continuous_integration/environment-3.9-jdk8-dev.yaml +++ b/continuous_integration/environment-3.9-jdk8-dev.yaml @@ -19,7 +19,7 @@ dependencies: - pre-commit - prompt_toolkit - psycopg2 -- pyarrow>=1.0.1 +- pyarrow>=3.0.0 - pygments - pyhive - pytest-cov diff --git a/docker/conda.txt b/docker/conda.txt index 183597c22..79477e40d 100644 --- a/docker/conda.txt +++ b/docker/conda.txt @@ -13,7 +13,7 @@ tzlocal>=2.1 fastapi>=0.69.0 nest-asyncio>=1.4.3 uvicorn>=0.11.3 -pyarrow>=1.0.1 +pyarrow>=3.0.0 prompt_toolkit>=3.0.8 pygments>=2.7.1 dask-ml>=2022.1.22 diff --git a/docker/main.dockerfile b/docker/main.dockerfile index 21eaf6df2..cae7fb96e 100644 --- a/docker/main.dockerfile +++ b/docker/main.dockerfile @@ -13,7 +13,7 @@ RUN conda config --add channels conda-forge \ "tzlocal>=2.1" \ "fastapi>=0.69.0" \ "uvicorn>=0.11.3" \ - "pyarrow>=1.0.1" \ + "pyarrow>=3.0.0" \ "prompt_toolkit>=3.0.8" \ "pygments>=2.7.1" \ "dask-ml>=2022.1.22" \ diff --git a/setup.py b/setup.py index d1ff97527..e2bc1d540 100755 --- a/setup.py +++ b/setup.py @@ -104,7 +104,7 @@ def build(self): "pytest-cov>=2.10.1", "mock>=4.0.3", "sphinx>=3.2.1", - "pyarrow>=1.0.1", + "pyarrow>=3.0.0", "dask-ml>=2022.1.22", "scikit-learn>=1.0.0", "intake>=0.6.0",