From 96aaa0504ad215196e8e53126f3de5486e99f42e Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 7 May 2024 14:03:21 +0000 Subject: [PATCH 1/6] test --- .github/workflows/pr.yaml | 2 +- ci/cudf_pandas_scripts/pandas-tests/diff.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f9d5976f1fe..3f188665257 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -181,7 +181,7 @@ jobs: test_summary_show: "none" pandas-tests-diff: # diff the results of running the Pandas unit tests and publish a job summary - needs: pandas-tests + # needs: pandas-tests uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.06 with: node_type: cpu4 diff --git a/ci/cudf_pandas_scripts/pandas-tests/diff.sh b/ci/cudf_pandas_scripts/pandas-tests/diff.sh index f87a3a36fcc..87cb89cb004 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/diff.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/diff.sh @@ -17,8 +17,8 @@ MAIN_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.main-${RAPIDS_FULL_VER PR_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.pr-${RAPIDS_FULL_VERSION}-results.json rapids-logger "Fetching latest available results from nightly" -aws s3api list-objects-v2 --bucket rapids-downloads --prefix "nightly/" --query "sort_by(Contents[?ends_with(Key, '_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json')], &LastModified)[::-1].[Key]" --output text > s3_output.txt - +aws s3api list-objects-v2 --bucket rapids-downloads --prefix "nightly/" --query "sort_by(Contents[?ends_with(Key, '_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json')], &LastModified)[::].[Key]" --output text > s3_output.txt +cat s3_output.txtx read -r COMPARE_ENV < s3_output.txt export COMPARE_ENV rapids-logger "Latest available results from nightly: ${COMPARE_ENV}" From eda6883b65787f348222ca678555900652e5f600 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 7 May 2024 14:06:01 +0000 Subject: [PATCH 2/6] test --- ci/cudf_pandas_scripts/pandas-tests/diff.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cudf_pandas_scripts/pandas-tests/diff.sh b/ci/cudf_pandas_scripts/pandas-tests/diff.sh index 87cb89cb004..0ce0a63e097 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/diff.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/diff.sh @@ -18,7 +18,7 @@ PR_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.pr-${RAPIDS_FULL_VERSION rapids-logger "Fetching latest available results from nightly" aws s3api list-objects-v2 --bucket rapids-downloads --prefix "nightly/" --query "sort_by(Contents[?ends_with(Key, '_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json')], &LastModified)[::].[Key]" --output text > s3_output.txt -cat s3_output.txtx +cat s3_output.txt read -r COMPARE_ENV < s3_output.txt export COMPARE_ENV rapids-logger "Latest available results from nightly: ${COMPARE_ENV}" From e7e2bcfc996082130c8cbeea8efdb4134778764f Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 7 May 2024 14:16:11 +0000 Subject: [PATCH 3/6] test --- ci/cudf_pandas_scripts/pandas-tests/diff.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ci/cudf_pandas_scripts/pandas-tests/diff.sh b/ci/cudf_pandas_scripts/pandas-tests/diff.sh index 0ce0a63e097..60687a2d772 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/diff.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/diff.sh @@ -19,10 +19,13 @@ PR_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.pr-${RAPIDS_FULL_VERSION rapids-logger "Fetching latest available results from nightly" aws s3api list-objects-v2 --bucket rapids-downloads --prefix "nightly/" --query "sort_by(Contents[?ends_with(Key, '_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json')], &LastModified)[::].[Key]" --output text > s3_output.txt cat s3_output.txt -read -r COMPARE_ENV < s3_output.txt -export COMPARE_ENV +# read -r COMPARE_ENV < s3_output.txt +# export COMPARE_ENV +COMPARE_ENV=$(head -1 s3_output.txt) rapids-logger "Latest available results from nightly: ${COMPARE_ENV}" +rapids-logger s3_output.txt + aws s3 cp "s3://rapids-downloads/${COMPARE_ENV}" main-results.json aws s3 cp $PR_ARTIFACT pr-results.json From ce319eb0cfc0cca698eff89dde810944103efd44 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 7 May 2024 14:20:42 +0000 Subject: [PATCH 4/6] test --- ci/cudf_pandas_scripts/pandas-tests/diff.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cudf_pandas_scripts/pandas-tests/diff.sh b/ci/cudf_pandas_scripts/pandas-tests/diff.sh index 60687a2d772..e315b01a36a 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/diff.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/diff.sh @@ -21,7 +21,7 @@ aws s3api list-objects-v2 --bucket rapids-downloads --prefix "nightly/" --query cat s3_output.txt # read -r COMPARE_ENV < s3_output.txt # export COMPARE_ENV -COMPARE_ENV=$(head -1 s3_output.txt) +COMPARE_ENV=$(tail -n 1 s3_output.txt) rapids-logger "Latest available results from nightly: ${COMPARE_ENV}" rapids-logger s3_output.txt From f6b8c00748c838ebe85edaf217fd5e2981d5073d Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 7 May 2024 14:24:10 +0000 Subject: [PATCH 5/6] Fix ci to fetch latest results --- .github/workflows/pr.yaml | 2 +- ci/cudf_pandas_scripts/pandas-tests/diff.sh | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3f188665257..f9d5976f1fe 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -181,7 +181,7 @@ jobs: test_summary_show: "none" pandas-tests-diff: # diff the results of running the Pandas unit tests and publish a job summary - # needs: pandas-tests + needs: pandas-tests uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.06 with: node_type: cpu4 diff --git a/ci/cudf_pandas_scripts/pandas-tests/diff.sh b/ci/cudf_pandas_scripts/pandas-tests/diff.sh index e315b01a36a..4bbdaa09197 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/diff.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/diff.sh @@ -19,13 +19,9 @@ PR_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.pr-${RAPIDS_FULL_VERSION rapids-logger "Fetching latest available results from nightly" aws s3api list-objects-v2 --bucket rapids-downloads --prefix "nightly/" --query "sort_by(Contents[?ends_with(Key, '_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json')], &LastModified)[::].[Key]" --output text > s3_output.txt cat s3_output.txt -# read -r COMPARE_ENV < s3_output.txt -# export COMPARE_ENV COMPARE_ENV=$(tail -n 1 s3_output.txt) rapids-logger "Latest available results from nightly: ${COMPARE_ENV}" -rapids-logger s3_output.txt - aws s3 cp "s3://rapids-downloads/${COMPARE_ENV}" main-results.json aws s3 cp $PR_ARTIFACT pr-results.json From 850253b3af1197ddc2e913646289b7e3b855bc80 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 7 May 2024 09:54:27 -0500 Subject: [PATCH 6/6] Update diff.sh Co-authored-by: Ray Douglass <3107146+raydouglass@users.noreply.github.com> --- ci/cudf_pandas_scripts/pandas-tests/diff.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/cudf_pandas_scripts/pandas-tests/diff.sh b/ci/cudf_pandas_scripts/pandas-tests/diff.sh index 4bbdaa09197..6cf70a2347f 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/diff.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/diff.sh @@ -17,8 +17,7 @@ MAIN_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.main-${RAPIDS_FULL_VER PR_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.pr-${RAPIDS_FULL_VERSION}-results.json rapids-logger "Fetching latest available results from nightly" -aws s3api list-objects-v2 --bucket rapids-downloads --prefix "nightly/" --query "sort_by(Contents[?ends_with(Key, '_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json')], &LastModified)[::].[Key]" --output text > s3_output.txt -cat s3_output.txt +aws s3api list-objects-v2 --bucket rapids-downloads --prefix "nightly/" --query "sort_by(Contents[?ends_with(Key, '_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json')], &LastModified)[::].[Key]" --output text | tee s3_output.txt COMPARE_ENV=$(tail -n 1 s3_output.txt) rapids-logger "Latest available results from nightly: ${COMPARE_ENV}"