Skip to content

Commit

Permalink
Backport of CI related PRs on 1.4.x (#46559)
Browse files Browse the repository at this point in the history
  • Loading branch information
simonjayhawkins authored Mar 29, 2022
1 parent eb27f7b commit 749872f
Show file tree
Hide file tree
Showing 26 changed files with 165 additions and 84 deletions.
2 changes: 1 addition & 1 deletion .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ jobs:
environment:
ENV_FILE: ci/deps/circle-38-arm64.yaml
PYTEST_WORKERS: auto
PATTERN: "not slow and not network and not clipboard and not arm_slow"
PATTERN: "not single_cpu and not slow and not network and not clipboard and not arm_slow and not db"
PYTEST_TARGET: "pandas"
steps:
- checkout
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/posix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,7 @@ jobs:
shell: bash
run: |
# TODO: re-enable cov, its slowing the tests down though
# TODO: Unpin Cython, the new Cython 0.29.26 is causing compilation errors
pip install Cython==0.29.25 numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3
pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 pytest-asyncio>=0.17 hypothesis>=5.5.3
if: ${{ env.IS_PYPY == 'true' }}

- name: Build Pandas
Expand Down
2 changes: 1 addition & 1 deletion azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
/opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
. ~/virtualenvs/pandas-dev/bin/activate && \
python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \
pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
python setup.py build_ext -q -j2 && \
python -m pip install --no-build-isolation -e . && \
pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml"
Expand Down
1 change: 1 addition & 0 deletions ci/deps/actions-310-numpydev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ dependencies:
- pytest-cov
- pytest-xdist>=1.31
- hypothesis>=5.5.3
- pytest-asyncio>=0.17

# pandas dependencies
- python-dateutil
Expand Down
4 changes: 4 additions & 0 deletions ci/deps/actions-310.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ dependencies:
- pytest-xdist>=1.31
- hypothesis>=5.5.3
- psutil
- pytest-asyncio>=0.17
- boto3

# required dependencies
- python-dateutil
Expand All @@ -21,6 +23,7 @@ dependencies:
- beautifulsoup4
- blosc
- bottleneck
- brotlipy
- fastparquet
- fsspec
- html5lib
Expand All @@ -39,6 +42,7 @@ dependencies:
- pytables
- pyarrow
- pyreadstat
- python-snappy
- pyxlsb
- s3fs
- scipy
Expand Down
11 changes: 6 additions & 5 deletions ci/deps/actions-38-downstream_compat.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ dependencies:
- pytest-xdist>=1.31
- hypothesis>=5.5.3
- psutil
- pytest-asyncio>=0.17
- boto3

# required dependencies
- python-dateutil
Expand All @@ -21,6 +23,7 @@ dependencies:
# optional dependencies
- beautifulsoup4
- blosc
- brotlipy
- bottleneck
- fastparquet
- fsspec
Expand All @@ -35,10 +38,11 @@ dependencies:
- odfpy
- pandas-gbq
- psycopg2
- pymysql
- pytables
- pyarrow
- pymysql
- pyreadstat
- pytables
- python-snappy
- pyxlsb
- s3fs
- scipy
Expand All @@ -52,17 +56,14 @@ dependencies:

# downstream packages
- aiobotocore
- boto3
- botocore
- cftime
- dask
- ipython
- geopandas
- python-snappy
- seaborn
- scikit-learn
- statsmodels
- brotlipy
- coverage
- pandas-datareader
- pyyaml
Expand Down
8 changes: 6 additions & 2 deletions ci/deps/actions-38-minimum_versions.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ dependencies:
- pytest-xdist>=1.31
- hypothesis>=5.5.3
- psutil
- pytest-asyncio>=0.17
- boto3

# required dependencies
- python-dateutil=2.8.1
Expand All @@ -23,6 +25,7 @@ dependencies:
- beautifulsoup4=4.8.2
- blosc=1.20.1
- bottleneck=1.3.1
- brotlipy=0.7.0
- fastparquet=0.4.0
- fsspec=0.7.4
- html5lib=1.1
Expand All @@ -37,10 +40,11 @@ dependencies:
- openpyxl=3.0.3
- pandas-gbq=0.14.0
- psycopg2=2.8.4
- pymysql=0.10.1
- pytables=3.6.1
- pyarrow=1.0.1
- pymysql=0.10.1
- pyreadstat=1.1.0
- pytables=3.6.1
- python-snappy=0.6.0
- pyxlsb=1.0.6
- s3fs=0.4.0
- scipy=1.4.1
Expand Down
8 changes: 6 additions & 2 deletions ci/deps/actions-38.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ dependencies:
- pytest-xdist>=1.31
- hypothesis>=5.5.3
- psutil
- pytest-asyncio>=0.17
- boto3

# required dependencies
- python-dateutil
Expand All @@ -21,6 +23,7 @@ dependencies:
- beautifulsoup4
- blosc
- bottleneck
- brotlipy
- fastparquet
- fsspec
- html5lib
Expand All @@ -34,10 +37,11 @@ dependencies:
- odfpy
- pandas-gbq
- psycopg2
- pymysql
- pytables
- pyarrow
- pymysql
- pyreadstat
- pytables
- python-snappy
- pyxlsb
- s3fs
- scipy
Expand Down
6 changes: 5 additions & 1 deletion ci/deps/actions-39.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ dependencies:
- pytest-xdist>=1.31
- hypothesis>=5.5.3
- psutil
- pytest-asyncio>=0.17
- boto3

# required dependencies
- python-dateutil
Expand All @@ -21,6 +23,7 @@ dependencies:
- beautifulsoup4
- blosc
- bottleneck
- brotlipy
- fastparquet
- fsspec
- html5lib
Expand All @@ -35,9 +38,10 @@ dependencies:
- pandas-gbq
- psycopg2
- pymysql
- pytables
- pyarrow
- pyreadstat
- pytables
- python-snappy
- pyxlsb
- s3fs
- scipy
Expand Down
50 changes: 42 additions & 8 deletions ci/deps/circle-38-arm64.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,52 @@ channels:
dependencies:
- python=3.8

# tools
- cython>=0.29.24
# test dependencies
- cython=0.29.24
- pytest>=6.0
- pytest-cov
- pytest-xdist>=1.31
- hypothesis>=5.5.3
- psutil
- pytest-asyncio>=0.17
- boto3

# pandas dependencies
- botocore>=1.11
- flask
- moto
- numpy
# required dependencies
- python-dateutil
- numpy
- pytz

# optional dependencies
- beautifulsoup4
- blosc
- bottleneck
- brotlipy
- fastparquet
- fsspec
- html5lib
- gcsfs
- jinja2
- lxml
- matplotlib
- numba
- numexpr
- openpyxl
- odfpy
- pandas-gbq
- psycopg2
- pyarrow
- pymysql
# Not provided on ARM
#- pyreadstat
- pytables
- python-snappy
- pyxlsb
- s3fs
- scipy
- sqlalchemy
- tabulate
- xarray
- xlrd
- xlsxwriter
- xlwt
- zstandard
- pip
4 changes: 3 additions & 1 deletion doc/source/getting_started/install.rst
Original file line number Diff line number Diff line change
Expand Up @@ -410,5 +410,7 @@ Compression
========================= ================== =============================================================
Dependency Minimum Version Notes
========================= ================== =============================================================
Zstandard Zstandard compression
brotli 0.7.0 Brotli compression
python-snappy 0.6.0 Snappy compression
Zstandard 0.15.2 Zstandard compression
========================= ================== =============================================================
2 changes: 1 addition & 1 deletion environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ dependencies:
- pytest>=6.0
- pytest-cov
- pytest-xdist>=1.31
- pytest-asyncio
- pytest-asyncio>=0.17
- pytest-instafail

# downstream tests
Expand Down
17 changes: 14 additions & 3 deletions pandas/compat/_optional.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"bs4": "4.8.2",
"blosc": "1.20.1",
"bottleneck": "1.3.1",
"brotli": "0.7.0",
"fastparquet": "0.4.0",
"fsspec": "0.7.4",
"html5lib": "1.1",
Expand All @@ -34,6 +35,7 @@
"pyxlsb": "1.0.6",
"s3fs": "0.4.0",
"scipy": "1.4.1",
"snappy": "0.6.0",
"sqlalchemy": "1.4.0",
"tables": "3.6.1",
"tabulate": "0.8.7",
Expand All @@ -50,12 +52,14 @@
INSTALL_MAPPING = {
"bs4": "beautifulsoup4",
"bottleneck": "Bottleneck",
"brotli": "brotlipy",
"jinja2": "Jinja2",
"lxml.etree": "lxml",
"odf": "odfpy",
"pandas_gbq": "pandas-gbq",
"tables": "pytables",
"snappy": "python-snappy",
"sqlalchemy": "SQLAlchemy",
"jinja2": "Jinja2",
"tables": "pytables",
}


Expand All @@ -66,6 +70,13 @@ def get_version(module: types.ModuleType) -> str:
version = getattr(module, "__VERSION__", None)

if version is None:
if module.__name__ == "brotli":
# brotli doesn't contain attributes to confirm it's version
return ""
if module.__name__ == "snappy":
# snappy doesn't contain attributes to confirm it's version
# See https://github.com/andrix/python-snappy/pull/119
return ""
raise ImportError(f"Can't determine version for {module.__name__}")
if module.__name__ == "psycopg2":
# psycopg2 appends " (dt dec pq3 ext lo64)" to it's version
Expand Down Expand Up @@ -141,7 +152,7 @@ def import_optional_dependency(
minimum_version = min_version if min_version is not None else VERSIONS.get(parent)
if minimum_version:
version = get_version(module_to_get)
if Version(version) < Version(minimum_version):
if version and Version(version) < Version(minimum_version):
msg = (
f"Pandas requires version '{minimum_version}' or newer of '{parent}' "
f"(version '{version}' currently installed)."
Expand Down
2 changes: 1 addition & 1 deletion pandas/tests/arrays/string_/test_string.py
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,7 @@ def test_memory_usage(dtype):
# GH 33963

if dtype.storage == "pyarrow":
pytest.skip("not applicable")
pytest.skip(f"not applicable for {dtype.storage}")

series = pd.Series(["a", "b", "c"], dtype=dtype)

Expand Down
9 changes: 9 additions & 0 deletions pandas/tests/extension/arrow/test_bool.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
import numpy as np
import pytest

from pandas.compat import (
is_ci_environment,
is_platform_windows,
)

import pandas as pd
import pandas._testing as tm
from pandas.api.types import is_bool_dtype
Expand Down Expand Up @@ -91,6 +96,10 @@ def test_reduce_series_boolean(self):
pass


@pytest.mark.skipif(
is_ci_environment() and is_platform_windows(),
reason="Causes stack overflow on Windows CI",
)
class TestReduceBoolean(base.BaseBooleanReduceTests):
pass

Expand Down
17 changes: 11 additions & 6 deletions pandas/tests/extension/base/ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,17 +114,22 @@ def test_add_series_with_extension_array(self, data):
self.assert_series_equal(result, expected)

@pytest.mark.parametrize("box", [pd.Series, pd.DataFrame])
def test_direct_arith_with_ndframe_returns_not_implemented(self, data, box):
def test_direct_arith_with_ndframe_returns_not_implemented(
self, request, data, box
):
# EAs should return NotImplemented for ops with Series/DataFrame
# Pandas takes care of unboxing the series and calling the EA's op.
other = pd.Series(data)
if box is pd.DataFrame:
other = other.to_frame()
if hasattr(data, "__add__"):
result = data.__add__(other)
assert result is NotImplemented
else:
raise pytest.skip(f"{type(data).__name__} does not implement add")
if not hasattr(data, "__add__"):
request.node.add_marker(
pytest.mark.xfail(
reason=f"{type(data).__name__} does not implement add"
)
)
result = data.__add__(other)
assert result is NotImplemented


class BaseComparisonOpsTests(BaseOpsUtil):
Expand Down
Loading

0 comments on commit 749872f

Please sign in to comment.