Enable testing cudf.pandas unit tests for all minor versions of pandas

.github/workflows/pr.yaml

@@ -32,6 +32,8 @@ jobs:
       - wheel-tests-dask-cudf
       - devcontainer
       - unit-tests-cudf-pandas
+      - fetch-pandas-versions
+      - cross-pandas-version-tests
       - pandas-tests
       - pandas-tests-diff
     secrets: inherit
@@ -41,6 +43,40 @@ jobs:
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       enable_check_generated_files: false
+  fetch-pandas-versions:
+    runs-on: ubuntu-latest
+    outputs:
+      pandas-versions: ${{ steps.get-versions.outputs.versions }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.11'
+
+      - name: Install requests
+        run: pip install requests
+      - name: Install packaging
+        run: pip install packaging
+      - name: Fetch pandas versions
+        id: get-versions
+        run: |
+          versions=$(python ci/cudf_pandas_scripts/fetch_pandas_versions.py ">=2.0,<2.2.3dev0")
+          echo "::set-output name=versions::${versions}"
+  cross-pandas-version-tests:
+    needs: [fetch-pandas-versions, wheel-build-cudf]
+    secrets: inherit
+    strategy:
+      matrix:
+        pandas-version: ${{ fromJson(needs.fetch-pandas-versions.outputs.pandas-versions) }}
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      # This selects "ARCH=amd64 + the latest supported Python + CUDA".
+      matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
+      build_type: pull-request
+      script: ci/cudf_pandas_scripts/run_tests.sh ${{ matrix.pandas-version }}
   conda-cpp-build:
     needs: checks
     secrets: inherit

ci/cudf_pandas_scripts/fetch_pandas_versions.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+import requests
+from packaging.version import Version
+from packaging.specifiers import SpecifierSet
+import argparse
+
+def get_pandas_versions(pandas_range):
+    url = "https://pypi.org/pypi/pandas/json"
+    response = requests.get(url)
+    data = response.json()
+    versions = data['releases'].keys()
+
+    specifier = SpecifierSet(pandas_range)
+
+    minor_versions = list(set([version[:3] for version in versions if Version(version) in specifier]))
+
+    return minor_versions
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Filter pandas versions by prefix.")
+    parser.add_argument("pandas_range", type=str, help="The version prefix to filter by.")
+    args = parser.parse_args()
+
+    versions = get_pandas_versions(args.pandas_range)
+    print(versions)

ci/cudf_pandas_scripts/run_tests.sh

@@ -11,11 +11,12 @@ mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
 
 # Function to display script usage
 function display_usage {
-    echo "Usage: $0 [--no-cudf]"
+    echo "Usage: $0 [--no-cudf] [pandas-version]"
 }
 
 # Default value for the --no-cudf option
 no_cudf=false
+PANDAS_VERSION=""
 
 # Parse command-line arguments
 while [[ $# -gt 0 ]]; do
@@ -25,9 +26,14 @@ while [[ $# -gt 0 ]]; do
             shift
             ;;
         *)
-            echo "Error: Unknown option $1"
-            display_usage
-            exit 1
+            if [[ -z "$PANDAS_VERSION" ]]; then
+                PANDAS_VERSION=$1
+                shift
+            else
+                echo "Error: Unknown option $1"
+                display_usage
+                exit 1
+            fi
             ;;
     esac
 done
@@ -47,6 +53,14 @@ else
         "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)"
 fi
 
+# Conditionally install the specified version of pandas
+if [ -n "$PANDAS_VERSION" ]; then
+    echo "Installing pandas version: $PANDAS_VERSION"
+    python -m pip install pandas==$PANDAS_VERSION
+else
+    echo "No pandas version specified, using existing pandas installation"
+fi
+
 python -m pytest -p cudf.pandas \
     --cov-config=./python/cudf/.coveragerc \
     --cov=cudf \

python/cudf/cudf_pandas_tests/test_cudf_pandas.py

@@ -42,6 +42,8 @@
     get_calendar,
 )
 
+from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION
+
 # Accelerated pandas has the real pandas and cudf modules as attributes
 pd = xpd._fsproxy_slow
 cudf = xpd._fsproxy_fast
@@ -607,6 +609,10 @@ def test_array_function_series_fallback(series):
     tm.assert_equal(expect, got)
 
 
+@pytest.mark.skipif(
+    PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION,
+    reason="Fails in older versions of pandas",
+)
 def test_timedeltaproperties(series):
     psr, sr = series
     psr, sr = psr.astype("timedelta64[ns]"), sr.astype("timedelta64[ns]")
@@ -666,6 +672,10 @@ def test_maintain_container_subclasses(multiindex):
     assert isinstance(got, xpd.core.indexes.frozen.FrozenList)
 
 
+@pytest.mark.skipif(
+    PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION,
+    reason="Fails in older versions of pandas due to unsupported boxcar window type",
+)
 def test_rolling_win_type():
     pdf = pd.DataFrame(range(5))
     df = xpd.DataFrame(range(5))
@@ -1281,6 +1291,10 @@ def max_times_two(self):
     assert s.max_times_two() == 6
 
 
+@pytest.mark.skipif(
+    PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION,
+    reason="DatetimeArray.__floordiv__ missing in pandas-2.0.0",
+)
 def test_floordiv_array_vs_df():
     xarray = xpd.Series([1, 2, 3], dtype="datetime64[ns]").array
     parray = pd.Series([1, 2, 3], dtype="datetime64[ns]").array
@@ -1552,6 +1566,10 @@ def test_numpy_cupy_flatiter(series):
     assert type(arr.flat._fsproxy_slow) == np.flatiter
 
 
+@pytest.mark.skipif(
+    PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION,
+    reason="pyarrow_numpy storage type was not supported in pandas-2.0.0",
+)
 def test_arrow_string_arrays():
     cu_s = xpd.Series(["a", "b", "c"])
     pd_s = pd.Series(["a", "b", "c"])

python/cudf/cudf_pandas_tests/test_profiler.py

@@ -5,6 +5,8 @@
 import os
 import subprocess
 
+import pytest
+
 from cudf.pandas import LOADED, Profiler
 
 if not LOADED:
@@ -13,7 +15,13 @@
 import numpy as np
 import pandas as pd
 
+from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION
+
 
+@pytest.mark.skipif(
+    PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION,
+    reason="funciton names change across versions of pandas, so making sure it only runs on latest version of pandas",
+)
 def test_profiler():
     np.random.seed(42)
     with Profiler() as profiler: