From 8ee78ae641afb7d3d45eb424943ba7204cc3e8d3 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 19 Sep 2024 19:41:39 +0000 Subject: [PATCH 1/2] Switch to using native traceback --- ci/test_python_cudf.sh | 10 +++++++--- ci/test_python_other.sh | 10 +++++++--- ci/test_wheel_cudf.sh | 3 +++ ci/test_wheel_cudf_polars.sh | 3 ++- ci/test_wheel_dask_cudf.sh | 4 ++++ 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/ci/test_python_cudf.sh b/ci/test_python_cudf.sh index ae34047e87f..adbe7543b25 100755 --- a/ci/test_python_cudf.sh +++ b/ci/test_python_cudf.sh @@ -19,6 +19,7 @@ pushd python/pylibcudf/pylibcudf/tests python -m pytest \ --cache-clear \ --dist=worksteal \ + --tb=native \ . popd @@ -30,7 +31,8 @@ rapids-logger "pytest cudf" --cov-config=../.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-coverage.xml" \ - --cov-report=term + --cov-report=term \ + --tb=native # Run benchmarks with both cudf and pandas to ensure compatibility is maintained. # Benchmarks are run in DEBUG_ONLY mode, meaning that only small data sizes are used. @@ -44,7 +46,8 @@ rapids-logger "pytest for cudf benchmarks" --cov-config=.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-benchmark-coverage.xml" \ - --cov-report=term + --cov-report=term \ + --tb=native rapids-logger "pytest for cudf benchmarks using pandas" ./ci/run_cudf_pandas_pytest_benchmarks.sh \ @@ -53,7 +56,8 @@ rapids-logger "pytest for cudf benchmarks using pandas" --cov-config=.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-benchmark-pandas-coverage.xml" \ - --cov-report=term + --cov-report=term \ + --tb=native rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/ci/test_python_other.sh b/ci/test_python_other.sh index 06a24773cae..e55ea60fa01 100755 --- a/ci/test_python_other.sh +++ b/ci/test_python_other.sh @@ -27,18 +27,21 @@ DASK_DATAFRAME__QUERY_PLANNING=True ./ci/run_dask_cudf_pytests.sh \ --cov-config=../.coveragerc \ --cov=dask_cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/dask-cudf-coverage.xml" \ - --cov-report=term + --cov-report=term \ + --tb=native rapids-logger "pytest dask_cudf (legacy)" DASK_DATAFRAME__QUERY_PLANNING=False ./ci/run_dask_cudf_pytests.sh \ --junitxml="${RAPIDS_TESTS_DIR}/junit-dask-cudf-legacy.xml" \ --numprocesses=8 \ --dist=loadscope \ + --tb=native \ . rapids-logger "pytest cudf_kafka" ./ci/run_cudf_kafka_pytests.sh \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-kafka.xml" + --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-kafka.xml" \ + --tb=native rapids-logger "pytest custreamz" ./ci/run_custreamz_pytests.sh \ @@ -48,7 +51,8 @@ rapids-logger "pytest custreamz" --cov-config=../.coveragerc \ --cov=custreamz \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/custreamz-coverage.xml" \ - --cov-report=term + --cov-report=term \ + --tb=native rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 28ded2f8e0f..330601b479d 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -39,7 +39,9 @@ rapids-logger "pytest pylibcudf" pushd python/pylibcudf/pylibcudf/tests python -m pytest \ --cache-clear \ + --numprocesses=8 \ --dist=worksteal \ + --tb=native . popd @@ -50,5 +52,6 @@ python -m pytest \ --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf.xml" \ --numprocesses=8 \ --dist=worksteal \ + --tb=native . popd diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 9844090258a..e630e032a50 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -57,7 +57,8 @@ set +e --cov cudf_polars \ --cov-fail-under=100 \ --cov-config=./pyproject.toml \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars.xml" + --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars.xml" \ + --tb=native trap ERR set -e diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 0d39807d56c..3dc13a8e0f0 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -41,6 +41,8 @@ pushd python/dask_cudf/dask_cudf DASK_DATAFRAME__QUERY_PLANNING=True python -m pytest \ --junitxml="${RAPIDS_TESTS_DIR}/junit-dask-cudf.xml" \ --numprocesses=8 \ + --dist=worksteal \ + --tb=native \ . popd @@ -50,5 +52,7 @@ pushd python/dask_cudf/dask_cudf DASK_DATAFRAME__QUERY_PLANNING=False python -m pytest \ --junitxml="${RAPIDS_TESTS_DIR}/junit-dask-cudf-legacy.xml" \ --numprocesses=8 \ + --dist=worksteal \ + --tb=native \ . popd From 52a29fca0fd5a98be719da7c1672bfcc1a196d11 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 19 Sep 2024 22:01:56 +0000 Subject: [PATCH 2/2] move to pytest.ini --- ci/test_python_cudf.sh | 10 +++------- ci/test_python_other.sh | 10 +++------- ci/test_wheel_cudf.sh | 2 -- ci/test_wheel_cudf_polars.sh | 3 +-- ci/test_wheel_dask_cudf.sh | 2 -- python/cudf/benchmarks/pytest.ini | 1 + python/cudf/cudf/tests/pytest.ini | 1 + .../third_party_integration_tests/tests/pytest.ini | 3 +++ python/cudf_kafka/cudf_kafka/tests/pytest.ini | 4 ++++ python/cudf_polars/tests/pytest.ini | 4 ++++ python/custreamz/custreamz/tests/pytest.ini | 4 ++++ python/dask_cudf/dask_cudf/tests/pytest.ini | 4 ++++ python/pylibcudf/pylibcudf/tests/pytest.ini | 1 + 13 files changed, 29 insertions(+), 20 deletions(-) create mode 100644 python/cudf_kafka/cudf_kafka/tests/pytest.ini create mode 100644 python/cudf_polars/tests/pytest.ini create mode 100644 python/custreamz/custreamz/tests/pytest.ini create mode 100644 python/dask_cudf/dask_cudf/tests/pytest.ini diff --git a/ci/test_python_cudf.sh b/ci/test_python_cudf.sh index adbe7543b25..ae34047e87f 100755 --- a/ci/test_python_cudf.sh +++ b/ci/test_python_cudf.sh @@ -19,7 +19,6 @@ pushd python/pylibcudf/pylibcudf/tests python -m pytest \ --cache-clear \ --dist=worksteal \ - --tb=native \ . popd @@ -31,8 +30,7 @@ rapids-logger "pytest cudf" --cov-config=../.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-coverage.xml" \ - --cov-report=term \ - --tb=native + --cov-report=term # Run benchmarks with both cudf and pandas to ensure compatibility is maintained. # Benchmarks are run in DEBUG_ONLY mode, meaning that only small data sizes are used. @@ -46,8 +44,7 @@ rapids-logger "pytest for cudf benchmarks" --cov-config=.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-benchmark-coverage.xml" \ - --cov-report=term \ - --tb=native + --cov-report=term rapids-logger "pytest for cudf benchmarks using pandas" ./ci/run_cudf_pandas_pytest_benchmarks.sh \ @@ -56,8 +53,7 @@ rapids-logger "pytest for cudf benchmarks using pandas" --cov-config=.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-benchmark-pandas-coverage.xml" \ - --cov-report=term \ - --tb=native + --cov-report=term rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/ci/test_python_other.sh b/ci/test_python_other.sh index e55ea60fa01..06a24773cae 100755 --- a/ci/test_python_other.sh +++ b/ci/test_python_other.sh @@ -27,21 +27,18 @@ DASK_DATAFRAME__QUERY_PLANNING=True ./ci/run_dask_cudf_pytests.sh \ --cov-config=../.coveragerc \ --cov=dask_cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/dask-cudf-coverage.xml" \ - --cov-report=term \ - --tb=native + --cov-report=term rapids-logger "pytest dask_cudf (legacy)" DASK_DATAFRAME__QUERY_PLANNING=False ./ci/run_dask_cudf_pytests.sh \ --junitxml="${RAPIDS_TESTS_DIR}/junit-dask-cudf-legacy.xml" \ --numprocesses=8 \ --dist=loadscope \ - --tb=native \ . rapids-logger "pytest cudf_kafka" ./ci/run_cudf_kafka_pytests.sh \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-kafka.xml" \ - --tb=native + --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-kafka.xml" rapids-logger "pytest custreamz" ./ci/run_custreamz_pytests.sh \ @@ -51,8 +48,7 @@ rapids-logger "pytest custreamz" --cov-config=../.coveragerc \ --cov=custreamz \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/custreamz-coverage.xml" \ - --cov-report=term \ - --tb=native + --cov-report=term rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 330601b479d..a701bfe15e0 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -41,7 +41,6 @@ python -m pytest \ --cache-clear \ --numprocesses=8 \ --dist=worksteal \ - --tb=native . popd @@ -52,6 +51,5 @@ python -m pytest \ --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf.xml" \ --numprocesses=8 \ --dist=worksteal \ - --tb=native . popd diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index e630e032a50..9844090258a 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -57,8 +57,7 @@ set +e --cov cudf_polars \ --cov-fail-under=100 \ --cov-config=./pyproject.toml \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars.xml" \ - --tb=native + --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars.xml" trap ERR set -e diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 3dc13a8e0f0..361a42ccda9 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -42,7 +42,6 @@ DASK_DATAFRAME__QUERY_PLANNING=True python -m pytest \ --junitxml="${RAPIDS_TESTS_DIR}/junit-dask-cudf.xml" \ --numprocesses=8 \ --dist=worksteal \ - --tb=native \ . popd @@ -53,6 +52,5 @@ DASK_DATAFRAME__QUERY_PLANNING=False python -m pytest \ --junitxml="${RAPIDS_TESTS_DIR}/junit-dask-cudf-legacy.xml" \ --numprocesses=8 \ --dist=worksteal \ - --tb=native \ . popd diff --git a/python/cudf/benchmarks/pytest.ini b/python/cudf/benchmarks/pytest.ini index db24415ef9e..187d91996b2 100644 --- a/python/cudf/benchmarks/pytest.ini +++ b/python/cudf/benchmarks/pytest.ini @@ -6,3 +6,4 @@ python_classes = Bench python_functions = bench_* markers = pandas_incompatible: mark a benchmark that cannot be run with pandas +addopts = --tb=native diff --git a/python/cudf/cudf/tests/pytest.ini b/python/cudf/cudf/tests/pytest.ini index 2136bca0e28..8a594794fac 100644 --- a/python/cudf/cudf/tests/pytest.ini +++ b/python/cudf/cudf/tests/pytest.ini @@ -14,3 +14,4 @@ filterwarnings = ignore:Passing a BlockManager to DataFrame is deprecated:DeprecationWarning # PerformanceWarning from cupy warming up the JIT cache ignore:Jitify is performing a one-time only warm-up to populate the persistent cache:cupy._util.PerformanceWarning +addopts = --tb=native diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/pytest.ini b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/pytest.ini index 817d98e6ba2..98459035298 100644 --- a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/pytest.ini +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/pytest.ini @@ -1,3 +1,5 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + [pytest] xfail_strict=true markers= @@ -5,3 +7,4 @@ markers= xfail_gold: this test is expected to fail in the gold pass xfail_cudf_pandas: this test is expected to fail in the cudf_pandas pass xfail_compare: this test is expected to fail in the comparison pass +addopts = --tb=native diff --git a/python/cudf_kafka/cudf_kafka/tests/pytest.ini b/python/cudf_kafka/cudf_kafka/tests/pytest.ini new file mode 100644 index 00000000000..7b0a9f29fb1 --- /dev/null +++ b/python/cudf_kafka/cudf_kafka/tests/pytest.ini @@ -0,0 +1,4 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +[pytest] +addopts = --tb=native diff --git a/python/cudf_polars/tests/pytest.ini b/python/cudf_polars/tests/pytest.ini new file mode 100644 index 00000000000..7b0a9f29fb1 --- /dev/null +++ b/python/cudf_polars/tests/pytest.ini @@ -0,0 +1,4 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +[pytest] +addopts = --tb=native diff --git a/python/custreamz/custreamz/tests/pytest.ini b/python/custreamz/custreamz/tests/pytest.ini new file mode 100644 index 00000000000..7b0a9f29fb1 --- /dev/null +++ b/python/custreamz/custreamz/tests/pytest.ini @@ -0,0 +1,4 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +[pytest] +addopts = --tb=native diff --git a/python/dask_cudf/dask_cudf/tests/pytest.ini b/python/dask_cudf/dask_cudf/tests/pytest.ini new file mode 100644 index 00000000000..7b0a9f29fb1 --- /dev/null +++ b/python/dask_cudf/dask_cudf/tests/pytest.ini @@ -0,0 +1,4 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +[pytest] +addopts = --tb=native diff --git a/python/pylibcudf/pylibcudf/tests/pytest.ini b/python/pylibcudf/pylibcudf/tests/pytest.ini index 1761c0f011c..f572f85ca49 100644 --- a/python/pylibcudf/pylibcudf/tests/pytest.ini +++ b/python/pylibcudf/pylibcudf/tests/pytest.ini @@ -6,3 +6,4 @@ filterwarnings = error ignore:::.*xdist.* ignore:::.*pytest.* +addopts = --tb=native