Replace stash-artifacts.{sh,py} -> manage-artifacts.py

Also:
* Remove publish-artifacts.sh
* Upload artifacts to /{commit_id}/ prefix

.github/workflows/jvm_tests.yml

@@ -40,12 +40,6 @@ jobs:
       - name: Log into Docker registry (AWS ECR)
         run: bash ops/pipeline/login-docker-registry.sh
       - run: bash ops/pipeline/build-jvm-manylinux2014.sh ${{ matrix.arch }}
-      - name: Upload libxgboost4j.so
-        run: |
-          libname=lib/libxgboost4j_linux_${{ matrix.arch }}_${{ github.sha }}.so
-          mv -v lib/libxgboost4j.so ${libname}
-          bash ops/pipeline/publish-artifact.sh ${libname} \
-            s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/libxgboost4j/
 
   build-jvm-gpu:
     name: Build libxgboost4j.so with CUDA
@@ -64,7 +58,10 @@ jobs:
       - run: bash ops/pipeline/build-jvm-gpu.sh
       - name: Stash files
         run: |
-          bash ops/pipeline/stash-artifacts.sh stash build-jvm-gpu lib/libxgboost4j.so
+          python3 ops/pipeline/manage-artifacts.py upload \
+            --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \
+            --prefix cache/${{ github.run_id }}/build-jvm-gpu \
+            lib/libxgboost4j.so
 
   build-jvm-mac:
     name: "Build libxgboost4j.dylib for ${{ matrix.description }}"
@@ -75,11 +72,11 @@ jobs:
         include:
           - description: "MacOS (Apple Silicon)"
             script: ops/pipeline/build-jvm-macos-apple-silicon.sh
-            libname: libxgboost4j_m1_${{ github.sha }}.dylib
+            libname: libxgboost4j_m1.dylib
             runner: macos-14
           - description: "MacOS (Intel)"
             script: ops/pipeline/build-jvm-macos-intel.sh
-            libname: libxgboost4j_intel_${{ github.sha }}.dylib
+            libname: libxgboost4j_intel.dylib
             runner: macos-13
     steps:
       - uses: actions/checkout@v4
@@ -89,8 +86,10 @@ jobs:
       - name: Upload libxgboost4j.dylib
         run: |
           mv -v lib/libxgboost4j.dylib ${{ matrix.libname }}
-          bash ops/pipeline/publish-artifact.sh ${{ matrix.libname }} \
-            s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/libxgboost4j/
+          python3 ops/pipeline/manage-artifacts.py upload \
+            --s3-bucket xgboost-nightly-builds \
+            --prefix ${{ env.BRANCH_NAME }}/${{ github.sha }} --make-public \
+            ${{ matrix.libname }}
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }}
@@ -112,13 +111,18 @@ jobs:
         run: bash ops/pipeline/login-docker-registry.sh
       - name: Unstash files
         run: |
-          bash ops/pipeline/stash-artifacts.sh unstash build-jvm-gpu lib/libxgboost4j.so
+          python3 ops/pipeline/manage-artifacts.py download \
+            --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \
+            --prefix cache/${{ github.run_id }}/build-jvm-gpu \
+            --dest-dir lib \
+            libxgboost4j.so
       - run: bash ops/pipeline/build-jvm-doc.sh
       - name: Upload JVM doc
         run: |
-          bash ops/pipeline/publish-artifact.sh \
-            jvm-packages/${{ env.BRANCH_NAME }}.tar.bz2 \
-            s3://xgboost-docs/
+          python3 ops/pipeline/manage-artifacts.py upload \
+            --s3-bucket xgboost-docs \
+            --prefix ${BRANCH_NAME}/${GITHUB_SHA} --make-public \
+            jvm-packages/${{ env.BRANCH_NAME }}.tar.bz2
 
   build-test-jvm-packages:
     name: Build and test JVM packages (Linux, Scala ${{ matrix.scala_version }})
@@ -144,8 +148,10 @@ jobs:
           SCALA_VERSION: ${{ matrix.scala_version }}
       - name: Stash files
         run: |
-          bash ops/pipeline/stash-artifacts.sh stash \
-            build-test-jvm-packages lib/libxgboost4j.so
+          python3 ops/pipeline/manage-artifacts.py upload \
+            --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \
+            --prefix cache/${{ github.run_id }}/build-test-jvm-packages \
+            lib/libxgboost4j.so
         if: matrix.scala_version == '2.13'
 
   build-test-jvm-packages-other-os:
@@ -213,7 +219,11 @@ jobs:
         run: bash ops/pipeline/login-docker-registry.sh
       - name: Unstash files
         run: |
-          bash ops/pipeline/stash-artifacts.sh unstash build-jvm-gpu lib/libxgboost4j.so
+          python3 ops/pipeline/manage-artifacts.py download \
+            --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \
+            --prefix cache/${{ github.run_id }}/build-jvm-gpu \
+            --dest-dir lib \
+            libxgboost4j.so
       - run: bash ops/pipeline/test-jvm-gpu.sh
         env:
           SCALA_VERSION: ${{ matrix.scala_version }}
@@ -247,9 +257,11 @@ jobs:
         run: bash ops/pipeline/login-docker-registry.sh
       - name: Unstash files
         run: |
-          bash ops/pipeline/stash-artifacts.sh \
-            unstash ${{ matrix.variant.artifact_from }} \
-            lib/libxgboost4j.so
+          python3 ops/pipeline/manage-artifacts.py download \
+            --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \
+            --prefix cache/${{ github.run_id }}/${{ matrix.variant.artifact_from }} \
+            --dest-dir lib \
+            libxgboost4j.so
           ls -lh lib/libxgboost4j.so
       - name: Deploy JVM packages to S3
         run: |

.github/workflows/main.yml

@@ -30,7 +30,11 @@ jobs:
         run: bash ops/pipeline/login-docker-registry.sh
       - run: bash ops/pipeline/build-cpu.sh
       - name: Stash CLI executable
-        run: bash ops/pipeline/stash-artifacts.sh stash build-cpu ./xgboost
+        run: |
+          python3 ops/pipeline/manage-artifacts.py upload \
+            --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \
+            --prefix cache/${{ github.run_id }}/build-cpu \
+            ./xgboost
 
   build-cpu-arm64:
     name: Build CPU ARM64 + manylinux_2_28_aarch64 wheel
@@ -49,12 +53,10 @@ jobs:
       - run: bash ops/pipeline/build-cpu-arm64.sh
       - name: Stash files
         run: |
-          bash ops/pipeline/stash-artifacts.sh stash build-cpu-arm64 \
-          ./xgboost python-package/dist/*.whl
-      - name: Upload Python wheel
-        run: |
-          bash ops/pipeline/publish-artifact.sh python-package/dist/*.whl \
-            s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/
+          python3 ops/pipeline/manage-artifacts.py upload \
+            --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \
+            --prefix cache/${{ github.run_id }}/build-cpu-arm64 \
+            ./xgboost python-package/dist/*.whl
 
   build-cuda:
     name: Build CUDA + manylinux_2_28_x86_64 wheel
@@ -74,15 +76,10 @@ jobs:
           bash ops/pipeline/build-cuda.sh xgb-ci.gpu_build_rockylinux8 disable-rmm
       - name: Stash files
         run: |
-          bash ops/pipeline/stash-artifacts.sh stash build-cuda \
+          python3 ops/pipeline/manage-artifacts.py upload \
+            --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \
+            --prefix cache/${{ github.run_id }}/build-cuda \
             build/testxgboost ./xgboost python-package/dist/*.whl
-      - name: Upload Python wheel
-        run: |
-          for file in python-package/dist/*.whl python-package/dist/meta.json
-          do
-            bash ops/pipeline/publish-artifact.sh "${file}" \
-              s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/
-          done
 
   build-cuda-with-rmm:
     name: Build CUDA with RMM
@@ -102,12 +99,10 @@ jobs:
           bash ops/pipeline/build-cuda.sh xgb-ci.gpu_build_rockylinux8 enable-rmm
       - name: Stash files
         run: |
-          bash ops/pipeline/stash-artifacts.sh  \
-            stash build-cuda-with-rmm build/testxgboost
-      - name: Upload Python wheel
-        run: |
-          bash ops/pipeline/publish-artifact.sh python-package/dist/*.whl \
-            s3://xgboost-nightly-builds/experimental_build_with_rmm/
+          python3 ops/pipeline/manage-artifacts.py upload \
+            --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \
+            --prefix cache/${{ github.run_id }}/build-cuda-with-rmm \
+            build/testxgboost
 
   build-cuda-with-rmm-dev:
     name: Build CUDA with RMM (dev)
@@ -151,13 +146,6 @@ jobs:
       - name: Log into Docker registry (AWS ECR)
         run: bash ops/pipeline/login-docker-registry.sh
       - run: bash ops/pipeline/build-manylinux2014.sh ${{ matrix.arch }}
-      - name: Upload Python wheel
-        run: |
-          for wheel in python-package/dist/*.whl
-          do
-            bash ops/pipeline/publish-artifact.sh "${wheel}" \
-              s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/
-          done
 
   build-gpu-rpkg:
     name: Build GPU-enabled R package
@@ -174,10 +162,6 @@ jobs:
       - name: Log into Docker registry (AWS ECR)
         run: bash ops/pipeline/login-docker-registry.sh
       - run: bash ops/pipeline/build-gpu-rpkg.sh
-      - name: Upload R tarball
-        run: |
-          bash ops/pipeline/publish-artifact.sh xgboost_r_gpu_linux_*.tar.gz \
-            s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/
 
 
   test-cpp-gpu:
@@ -213,8 +197,11 @@ jobs:
         run: bash ops/pipeline/login-docker-registry.sh
       - name: Unstash gtest
         run: |
-          bash ops/pipeline/stash-artifacts.sh unstash ${{ matrix.artifact_from }}  \
-            build/testxgboost
+          python3 ops/pipeline/manage-artifacts.py download \
+            --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \
+            --prefix cache/${{ github.run_id }}/${{ matrix.artifact_from }} \
+            --dest-dir build \
+            testxgboost
           chmod +x build/testxgboost
       - run: bash ops/pipeline/test-cpp-gpu.sh ${{ matrix.suite }}
 
@@ -260,8 +247,12 @@ jobs:
         run: bash ops/pipeline/login-docker-registry.sh
       - name: Unstash Python wheel
         run: |
-          bash ops/pipeline/stash-artifacts.sh unstash ${{ matrix.artifact_from }} \
-            python-package/dist/*.whl ./xgboost
+          python3 ops/pipeline/manage-artifacts.py download \
+            --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \
+            --prefix cache/${{ github.run_id }}/${{ matrix.artifact_from }} \
+            --dest-dir wheelhouse \
+            *.whl xgboost
+          mv -v wheelhouse/xgboost .
           chmod +x ./xgboost
       - name: Run Python tests, ${{ matrix.description }}
         run: bash ops/pipeline/test-python-wheel.sh ${{ matrix.suite }} ${{ matrix.container }}

.github/workflows/windows.yml

@@ -30,9 +30,12 @@ jobs:
           submodules: "true"
       - run: powershell ops/pipeline/build-win64-gpu.ps1
       - name: Stash files
+        shell: powershell
         run: |
-          powershell ops/pipeline/stash-artifacts.ps1 stash build-win64-gpu `
-            build/testxgboost.exe xgboost.exe `
+          conda activate
+          python3 ops/pipeline/manage-artifacts.py upload \
+            --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \
+            --prefix cache/${{ github.run_id }}/build-win64-gpu \
             (Get-ChildItem python-package/dist/*.whl | Select-Object -Expand FullName)
 
   test-win64-gpu:
@@ -47,7 +50,12 @@ jobs:
         with:
           submodules: "true"
       - name: Unstash files
+        shell: powershell
         run: |
-          powershell ops/pipeline/stash-artifacts.ps1 unstash build-win64-gpu `
-            build/testxgboost.exe xgboost.exe python-package/dist/*.whl
+          conda activate
+          python3 ops/pipeline/manage-artifacts.py download \
+            --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \
+            --prefix cache/${{ github.run_id }}/build-win64-gpu \
+            --dest-dir python-package/dist \
+            *.whl
       - run: powershell ops/pipeline/test-win64-gpu.ps1

ops/pipeline/build-cpu-arm64.sh

@@ -39,3 +39,12 @@ if ! unzip -l ./python-package/dist/*.whl | grep libgomp > /dev/null; then
   echo "error: libgomp.so was not vendored in the wheel"
   exit -1
 fi
+
+echo "--- Upload Python wheel"
+if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
+then
+  python3 ops/pipeline/manage-artifacts.py upload \
+    --s3-bucket xgboost-nightly-builds \
+    --prefix ${BRANCH_NAME}/${GITHUB_SHA} --make-public \
+    python-package/dist/*.whl
+fi

ops/pipeline/build-cuda.sh

@@ -93,9 +93,9 @@ then
   echo "--- Upload Python wheel"
   if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
   then
-    aws s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
-      --acl public-read --no-progress
-    aws s3 cp python-package/dist/meta.json s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
-      --acl public-read --no-progress
+    python3 ops/pipeline/manage-artifacts.py upload \
+      --s3-bucket xgboost-nightly-builds \
+      --prefix ${BRANCH_NAME}/${GITHUB_SHA} --make-public \
+      python-package/dist/*.whl python-package/dist/meta.json
   fi
 fi

ops/pipeline/build-gpu-rpkg-impl.sh

@@ -33,4 +33,4 @@ cp -v lib/xgboost.so xgboost_rpack/src/
 echo 'all:' > xgboost_rpack/src/Makefile
 echo 'all:' > xgboost_rpack/src/Makefile.win
 mv xgboost_rpack/ xgboost/
-tar cvzf xgboost_r_gpu_linux_${commit_hash}.tar.gz xgboost/
+tar cvzf xgboost_r_gpu_linux.tar.gz xgboost/

ops/pipeline/build-gpu-rpkg.sh

@@ -8,6 +8,7 @@ then
   exit 1
 fi
 
+source ops/pipeline/classify-git-branch.sh
 source ops/pipeline/get-docker-registry-details.sh
 
 CONTAINER_TAG=${DOCKER_REGISTRY_URL}/xgb-ci.gpu_build_r_rockylinux8:main
@@ -18,3 +19,11 @@ python3 ops/docker_run.py \
   --container-tag ${CONTAINER_TAG} \
   -- ops/pipeline/build-gpu-rpkg-impl.sh \
   ${GITHUB_SHA}
+
+if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
+then
+  python3 ops/pipeline/manage-artifacts.py upload \
+    --s3-bucket xgboost-nightly-builds \
+    --prefix ${BRANCH_NAME}/${GITHUB_SHA} --make-public \
+    xgboost_r_gpu_linux.tar.gz
+fi

ops/pipeline/build-jvm-manylinux2014.sh

@@ -12,6 +12,7 @@ fi
 arch=$1
 container_id="xgb-ci.manylinux2014_${arch}"
 
+source ops/pipeline/classify-git-branch.sh
 source ops/pipeline/get-docker-registry-details.sh
 
 CONTAINER_TAG="${DOCKER_REGISTRY_URL}/${container_id}:main"
@@ -26,3 +27,13 @@ python3 ops/docker_run.py \
   "cd build && cmake .. -DJVM_BINDINGS=ON -DUSE_OPENMP=ON && make -j$(nproc)"
 ldd lib/libxgboost4j.so
 objdump -T lib/libxgboost4j.so | grep GLIBC_ | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu
+
+if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
+then
+  libname=lib/libxgboost4j_linux_${arch}.so
+  mv -v lib/libxgboost4j.so ${libname}
+  python3 ops/pipeline/manage-artifacts.py upload \
+    --s3-bucket xgboost-nightly-builds \
+    --prefix ${BRANCH_NAME}/${GITHUB_SHA} --make-public \
+    ${libname}
+fi

ops/pipeline/build-manylinux2014.sh

@@ -16,6 +16,7 @@ fi
 
 arch="$1"
 
+source ops/pipeline/classify-git-branch.sh
 source ops/pipeline/get-docker-registry-details.sh
 
 WHEEL_TAG="manylinux2014_${arch}"
@@ -65,3 +66,11 @@ python3 ops/script/rename_whl.py  \
   --platform-tag ${WHEEL_TAG}
 rm -v python-package/dist/xgboost_cpu-*.whl
 mv -v wheelhouse/xgboost_cpu-*.whl python-package/dist/
+
+if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
+then
+  python3 ops/pipeline/manage-artifacts.py upload \
+    --s3-bucket xgboost-nightly-builds \
+    --prefix ${BRANCH_NAME}/${GITHUB_SHA} --make-public \
+    python-package/dist/*.whl
+fi

ops/pipeline/build-win64-gpu.ps1

@@ -40,7 +40,9 @@ if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
 Write-Host "--- Upload Python wheel"
 cd ..
 if ( $is_release_branch -eq 1 ) {
-  aws s3 cp (Get-ChildItem python-package/dist/*.whl | Select-Object -Expand FullName) `
-    s3://xgboost-nightly-builds/$Env:BRANCH_NAME/ --acl public-read --no-progress
+  python ops/pipeline/manage-artifacts.py upload `
+    --s3-bucket 'xgboost-nightly-builds' `
+    --prefix "$Env:BRANCH_NAME/$Env:GITHUB_SHA" --make-public `
+    (Get-ChildItem python-package/dist/*.whl | Select-Object -Expand FullName)
   if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
 }