submodule-sync.sh

#!/bin/bash
#
# Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# NOTE:
#     this script is for jenkins only, and should not be used for local development
#     run with ci/Dockerfile in jenkins:
#         scl enable gcc-toolset-11 ci/submodule-sync.sh

set -ex

OWNER=${OWNER:-"NVIDIA"}
REPO=${REPO:-"spark-rapids-jni"}
PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
REPO_LOC="github.com/${OWNER}/${REPO}.git"

GIT_AUTHOR_NAME="spark-rapids automation"
GIT_COMMITTER_NAME="spark-rapids automation"
GIT_AUTHOR_EMAIL="70000568+nvauto@users.noreply.github.com"
GIT_COMMITTER_EMAIL="70000568+nvauto@users.noreply.github.com"
git submodule update --init --recursive

cudf_prev_sha=$(git -C thirdparty/cudf rev-parse HEAD)

INTERMEDIATE_HEAD=bot-submodule-sync-${REF}
# try cleanup remote first if no open PR for HEAD targeting BASE
$WORKSPACE/.github/workflows/action-helper/python/cleanup-bot-branch \
  --owner=${OWNER} --repo=${REPO} --head=${INTERMEDIATE_HEAD} --base=${REF} --token=${GIT_TOKEN} || true

remote_head=$(git ls-remote --heads origin ${INTERMEDIATE_HEAD})
if [[ -z $remote_head ]]; then
  git checkout -b ${INTERMEDIATE_HEAD} origin/${REF}
else
  git fetch origin ${INTERMEDIATE_HEAD} ${REF}
  git checkout -b ${INTERMEDIATE_HEAD} origin/${INTERMEDIATE_HEAD}
  git merge origin/${REF}
fi

# sync up cudf from remote, checkout to cudf release tag if CUDF_TAG is set
if [ -n "$CUDF_TAG" ]; then
  pushd thirdparty/cudf
  git checkout tags/$CUDF_TAG
  popd
else
  git submodule update --remote --merge
fi

cudf_pins_only=false
cudf_sha=$(git -C thirdparty/cudf rev-parse HEAD)
if [[ "${cudf_sha}" == "${cudf_prev_sha}" ]]; then
  echo "cuDF submodule is up to date. Try update cudf-pins..."
  cudf_pins_only=true
else
  echo "Try update cudf submodule to ${cudf_sha}..."
  git add .
  git commit -s -m "Update submodule cudf to ${cudf_sha}"
fi

echo "Build libcudf only to update pinned versions..."
MVN="mvn -Dmaven.wagon.http.retryHandler.count=3 -B"
set +e
# Don't do a full build. Just try to update/build CUDF with no patches on top of it.
# calling the antrun directly skips applying patches and also only builds
# libcudf
${MVN} antrun:run@build-libcudf ${MVN_MIRROR} \
  -DCPP_PARALLEL_LEVEL=${PARALLEL_LEVEL} \
  -Dlibcudf.build.configure=true \
  -Dlibcudf.dependency.mode=latest \
  -DUSE_GDS=ON \
  -DBUILD_TESTS=ON \
  -DUSE_SANITIZER=ON
validate_status=$?
set -e

LIBCUDF_BUILD_PATH=$(${MVN} help:evaluate -Dexpression=libcudf.build.path -q -DforceStdout)
# Extract the rapids-cmake sha1 that we need to pin too
rapids_cmake_sha=$(git -C ${LIBCUDF_BUILD_PATH}/_deps/rapids-cmake-src/ rev-parse HEAD)
echo "Update rapids-cmake pinned SHA1 to ${rapids_cmake_sha}"
echo "${rapids_cmake_sha}" > thirdparty/cudf-pins/rapids-cmake.sha

echo "Workaround for https://github.com/NVIDIA/spark-rapids-jni/issues/2582"
cudf_patch_path="cudf/cpp/cmake/thirdparty/patches"
sed -i "s|\${current_json_dir}|\${current_json_dir}/../${cudf_patch_path}|g" thirdparty/cudf-pins/versions.json

# Do the git add after the build so that we get
# the updated versions.json generated by the build
echo "Update cudf submodule to ${cudf_sha} with updated pinned versions"
git add .
if ! git diff-index --quiet HEAD; then
  # We perform a squash merge for submodule-sync commits
  git commit -s -m "Update pinned versions for cudf ${cudf_sha}"
elif ${cudf_pins_only}; then
  echo "No changes to commit. Exit early..."
  exit 0
fi

sha=$(git rev-parse HEAD)

echo "Test against ${cudf_sha}..."
set +e
# now build and test everything with the patches in place
${MVN} clean verify ${MVN_MIRROR} \
  -DCPP_PARALLEL_LEVEL=${PARALLEL_LEVEL} \
  -Dlibcudf.build.configure=true \
  -DUSE_GDS=ON -Dtest=*,!CuFileTest,!CudaFatalTest,!ColumnViewNonEmptyNullsTest \
  -DBUILD_TESTS=ON \
  -DUSE_SANITIZER=ON
verify_status=$?
set -e

test_pass="False"
if [[ ( "${verify_status}" == "0" ) && ( "${validate_status}" == "0" ) ]]; then
  echo "Test passed, will try merge the change"
  test_pass="True"
else
  echo "Test failed, will update the result"
fi

build_name=$(${MVN} help:evaluate -Dexpression=project.build.finalName -q -DforceStdout)
cuda_version=$(${MVN} help:evaluate -Dexpression=cuda.version -q -DforceStdout)
. ci/check-cuda-dependencies.sh "target/${build_name}-${cuda_version}.jar"

# push the intermediate branch and create PR against REF
# if test passed, it will try auto-merge the PR
# if test failed, it will only comment the test result in the PR
git push https://${GIT_USER}:${GIT_TOKEN}@${REPO_LOC} ${INTERMEDIATE_HEAD}
sleep 30 # sleep for a while to avoid inconsistent sha between HEAD branch and GitHub REST API
$WORKSPACE/.github/workflows/action-helper/python/submodule-sync \
  --owner=${OWNER} \
  --repo=${REPO} \
  --head=${INTERMEDIATE_HEAD} \
  --base=${REF} \
  --sha=${sha} \
  --cudf_sha=${cudf_sha} \
  --token=${GIT_TOKEN} \
  --passed=${test_pass} \
  --delete_head=True

exit $verify_status # always exit return code of mvn verify at the end