Merge remote-tracking branch 'upstream/branch-24.04' into branch-24.04

.github/workflows/auto-merge.yml

@@ -18,12 +18,12 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-      - branch-24.02
+      - branch-24.04
     types: [closed]
 
 env:
-  HEAD: branch-24.02
-  BASE: branch-24.04
+  HEAD: branch-24.04
+  BASE: branch-24.06
 
 jobs:
   auto-merge:

build/build-info

@@ -22,6 +22,7 @@
 #   git_path - The path to the repository
 #   libcudf_path - The path to the libcudf library
 set -e
+set -o pipefail
 
 echo_build_properties() {
   version=$1
@@ -34,7 +35,27 @@ echo_build_properties() {
   echo branch=$(cd "$git_path" && git rev-parse --abbrev-ref HEAD)
   echo date=$(date -u +%Y-%m-%dT%H:%M:%SZ)
   echo url=$(cd "$git_path" && git config --get remote.origin.url)
-  echo gpu_architectures=$(cuobjdump "$libcudf_path" 2>/dev/null | awk -F_ '/arch =/ {print $2}' | sort -n -u | tr '\n' ';')
+  gpu_architectures=$(
+    cuobjdump "$libcudf_path" | gawk '
+      match($0, /arch = sm_([0-9]+)/, tmp) {
+        arch[tmp[1]] = 1
+      }
+
+      END {
+        n = asorti(arch)
+        if (n == 0) {
+          print "ERROR: No fatbin ELF / PTX code sections found" > "/dev/stderr"
+          exit(1)
+        }
+        s = arch[1]
+        for (i = 2; i <= n; i++) {
+          s = s ";" arch[i]
+        }
+        print s
+      }
+    '
+  )
+  echo "gpu_architectures=$gpu_architectures"
   for arg in "$@"; do
     echo $arg
   done

ci/Jenkinsfile.premerge

@@ -234,6 +234,7 @@ git --no-pager diff --name-only HEAD \$BASE -- ${PREMERGE_DOCKERFILE} || true"""
                     // upload log only in case of build failure
                     def guardWords = ["gitlab.*?\\.com", "urm.*?\\.com"]
                     guardWords.add("nvidia-smi(?s)(.*?)(?=git)") // hide GPU info
+                    guardWords.add("sc-ipp*") // hide cloud info
                     githubHelper.uploadLogs(this, env.JOB_NAME, env.BUILD_NUMBER, null, guardWords)
 
                     githubHelper.updateCommitStatus("$BUILD_URL", "Fail", GitHubCommitState.FAILURE)

ci/submodule-sync.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -49,8 +49,14 @@ else
   git merge origin/${REF}
 fi
 
-# sync up cudf from remote
-git submodule update --remote --merge
+# sync up cudf from remote, checkout to cudf release tag if CUDF_TAG is set
+if [ -n "$CUDF_TAG" ]; then
+  pushd thirdparty/cudf
+  git checkout tags/$CUDF_TAG
+  popd
+else
+  git submodule update --remote --merge
+fi
 cudf_sha=$(git -C thirdparty/cudf rev-parse HEAD)
 if [[ "${cudf_sha}" == "${cudf_prev_sha}" ]]; then
   echo "Submodule is up to date."
@@ -59,7 +65,7 @@ fi
 
 echo "Try update cudf submodule to ${cudf_sha}..."
 git add .
-git diff-index --quiet HEAD || git commit -s -m "Update submodule cudf to ${cudf_sha}"
+git diff-index --quiet HEAD || git commit -s -m "Update submodule cudf to ${CUDF_TAG:-$cudf_sha}"
 sha=$(git rev-parse HEAD)
 
 echo "Test against ${cudf_sha}..."

src/main/cpp/CMakeLists.txt

@@ -94,6 +94,9 @@ include(cmake/Modules/ConfigureCUDA.cmake) # set other CUDA compilation flags
 # ##################################################################################################
 # * dependencies ----------------------------------------------------------------------------------
 
+# find NVTX
+include(${CUDF_DIR}/cpp/cmake/thirdparty/get_nvtx.cmake)
+
 # find CCCL
 include(${CUDF_DIR}/cpp/cmake/thirdparty/get_cccl.cmake)
 
@@ -224,6 +227,7 @@ target_link_libraries(
   -Wl,--whole-archive
     ${CUDFJNI_LIB}
     cudf::cudf
+    nvtx3-cpp
   -Wl,--no-whole-archive
     ${PARQUET_LIB}
     ${THRIFT_LIB}