diff --git a/ci/benchmark/build.sh b/ci/benchmark/build.sh index 8dd133c8fa3..07d6a0e50af 100755 --- a/ci/benchmark/build.sh +++ b/ci/benchmark/build.sh @@ -21,15 +21,15 @@ function hasArg { export PATH=/conda/bin:/usr/local/cuda/bin:$PATH export PARALLEL_LEVEL=4 export CUDA_REL=${CUDA_VERSION%.*} -export HOME=$WORKSPACE +export HOME="$WORKSPACE" # Parse git describe -cd $WORKSPACE +cd "$WORKSPACE" export GIT_DESCRIBE_TAG=`git describe --tags` export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` # Set Benchmark Vars -export GBENCH_BENCHMARKS_DIR=${WORKSPACE}/cpp/build/gbenchmarks/ +export GBENCH_BENCHMARKS_DIR="$WORKSPACE/cpp/build/gbenchmarks/" # Set `LIBCUDF_KERNEL_CACHE_PATH` environment variable to $HOME/.jitify-cache because # it's local to the container's virtual file system, and not shared with other CI jobs @@ -96,9 +96,9 @@ conda list --show-channel-urls logger "Build libcudf..." if [[ ${BUILD_MODE} == "pull-request" ]]; then - $WORKSPACE/build.sh clean libcudf cudf dask_cudf benchmarks tests --ptds + "$WORKSPACE/build.sh" clean libcudf cudf dask_cudf benchmarks tests --ptds else - $WORKSPACE/build.sh clean libcudf cudf dask_cudf benchmarks tests -l --ptds + "$WORKSPACE/build.sh" clean libcudf cudf dask_cudf benchmarks tests -l --ptds fi ################################################################################ @@ -144,9 +144,9 @@ function getReqs() { REQS=$(getReqs "${LIBCUDF_DEPS[@]}") -mkdir -p ${WORKSPACE}/tmp/benchmark -touch ${WORKSPACE}/tmp/benchmark/benchmarks.txt -ls ${GBENCH_BENCHMARKS_DIR} > ${WORKSPACE}/tmp/benchmark/benchmarks.txt +mkdir -p "$WORKSPACE/tmp/benchmark" +touch "$WORKSPACE/tmp/benchmark/benchmarks.txt" +ls ${GBENCH_BENCHMARKS_DIR} > "$WORKSPACE/tmp/benchmark/benchmarks.txt" #Disable error aborting while tests run, failed tests will not generate data logger "Running libcudf GBenchmarks..." @@ -161,13 +161,13 @@ do rm ./${BENCH}.json JOBEXITCODE=1 fi -done < ${WORKSPACE}/tmp/benchmark/benchmarks.txt +done < "$WORKSPACE/tmp/benchmark/benchmarks.txt" set -e -rm ${WORKSPACE}/tmp/benchmark/benchmarks.txt -cd ${WORKSPACE} -mv ${GBENCH_BENCHMARKS_DIR}/*.json ${WORKSPACE}/tmp/benchmark/ -python GBenchToASV.py -d ${WORKSPACE}/tmp/benchmark/ -t ${S3_ASV_DIR} -n libcudf -b branch-${MINOR_VERSION} -r "${REQS}" +rm "$WORKSPACE/tmp/benchmark/benchmarks.txt" +cd "$WORKSPACE" +mv ${GBENCH_BENCHMARKS_DIR}/*.json "$WORKSPACE/tmp/benchmark/" +python GBenchToASV.py -d "$WORKSPACE/tmp/benchmark/" -t ${S3_ASV_DIR} -n libcudf -b branch-${MINOR_VERSION} -r "${REQS}" ### # Run Python Benchmarks diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 6803d88d8dd..e11a0488624 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -10,7 +10,7 @@ export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} # Set home to the job's workspace -export HOME=$WORKSPACE +export HOME="$WORKSPACE" # Determine CUDA release version export CUDA_REL=${CUDA_VERSION%.*} @@ -21,10 +21,10 @@ export GPUCI_CONDA_RETRY_SLEEP=30 # Use Ninja to build, setup Conda Build Dir export CMAKE_GENERATOR="Ninja" -export CONDA_BLD_DIR="${WORKSPACE}/.conda-bld" +export CONDA_BLD_DIR="$WORKSPACE/.conda-bld" # Switch to project root; also root of repo checkout -cd $WORKSPACE +cd "$WORKSPACE" # If nightly build, append current YYMMDD to version if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh index 4f72f6dd772..40e80def8ae 100755 --- a/ci/cpu/upload.sh +++ b/ci/cpu/upload.sh @@ -29,8 +29,8 @@ fi gpuci_logger "Get conda file output locations" -export LIBCUDF_FILE=`conda build --no-build-id --croot ${WORKSPACE}/.conda-bld conda/recipes/libcudf --output` -export LIBCUDF_KAFKA_FILE=`conda build --no-build-id --croot ${WORKSPACE}/.conda-bld conda/recipes/libcudf_kafka --output` +export LIBCUDF_FILE=`conda build --no-build-id --croot "$WORKSPACE/.conda-bld" conda/recipes/libcudf --output` +export LIBCUDF_KAFKA_FILE=`conda build --no-build-id --croot "$WORKSPACE/.conda-bld" conda/recipes/libcudf_kafka --output` export CUDF_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/cudf --python=$PYTHON --output` export DASK_CUDF_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/dask-cudf --python=$PYTHON --output` export CUDF_KAFKA_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/cudf_kafka --python=$PYTHON --output` diff --git a/ci/docs/build.sh b/ci/docs/build.sh index bc2dc4eaba6..a7771124713 100755 --- a/ci/docs/build.sh +++ b/ci/docs/build.sh @@ -10,9 +10,9 @@ if [ -z "$PROJECT_WORKSPACE" ]; then exit 1 fi -export DOCS_WORKSPACE=$WORKSPACE/docs +export DOCS_WORKSPACE="$WORKSPACE/docs" export PATH=/conda/bin:/usr/local/cuda/bin:$PATH -export HOME=$WORKSPACE +export HOME="$WORKSPACE" export PROJECT_WORKSPACE=/rapids/cudf export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache" export PROJECTS=(cudf libcudf) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 7614e19cc89..e1ddfa1cc56 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -17,14 +17,14 @@ export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} # Set home to the job's workspace -export HOME=$WORKSPACE +export HOME="$WORKSPACE" # Switch to project root; also root of repo checkout -cd $WORKSPACE +cd "$WORKSPACE" # Determine CUDA release version export CUDA_REL=${CUDA_VERSION%.*} -export CONDA_ARTIFACT_PATH=${WORKSPACE}/ci/artifacts/cudf/cpu/.conda-bld/ +export CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/cudf/cpu/.conda-bld/" # Parse git describe export GIT_DESCRIBE_TAG=`git describe --tags` @@ -117,9 +117,9 @@ if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then gpuci_logger "Build from source" if [[ ${BUILD_MODE} == "pull-request" ]]; then - $WORKSPACE/build.sh clean libcudf cudf dask_cudf libcudf_kafka cudf_kafka benchmarks tests --ptds + "$WORKSPACE/build.sh" clean libcudf cudf dask_cudf libcudf_kafka cudf_kafka benchmarks tests --ptds else - $WORKSPACE/build.sh clean libcudf cudf dask_cudf libcudf_kafka cudf_kafka benchmarks tests -l --ptds + "$WORKSPACE/build.sh" clean libcudf cudf dask_cudf libcudf_kafka cudf_kafka benchmarks tests -l --ptds fi ################################################################################ @@ -140,12 +140,12 @@ if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then gpuci_logger "GoogleTests" set -x - cd $WORKSPACE/cpp/build + cd "$WORKSPACE/cpp/build" - for gt in ${WORKSPACE}/cpp/build/gtests/* ; do + for gt in "$WORKSPACE/cpp/build/gtests/"* ; do test_name=$(basename ${gt}) echo "Running GoogleTest $test_name" - ${gt} --gtest_output=xml:${WORKSPACE}/test-results/ + ${gt} --gtest_output=xml:"$WORKSPACE/test-results/" done fi else @@ -168,7 +168,7 @@ else for gt in gtests/* ; do test_name=$(basename ${gt}) echo "Running GoogleTest $test_name" - ${gt} --gtest_output=xml:${WORKSPACE}/test-results/ + ${gt} --gtest_output=xml:"$WORKSPACE/test-results/" done CUDF_CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "libcudf-*.tar.bz2"` @@ -185,9 +185,9 @@ else gpuci_logger "Build python libs from source" if [[ ${BUILD_MODE} == "pull-request" ]]; then - $WORKSPACE/build.sh cudf dask_cudf cudf_kafka --ptds + "$WORKSPACE/build.sh" cudf dask_cudf cudf_kafka --ptds else - $WORKSPACE/build.sh cudf dask_cudf cudf_kafka -l --ptds + "$WORKSPACE/build.sh" cudf dask_cudf cudf_kafka -l --ptds fi fi @@ -205,21 +205,21 @@ fi # TEST - Run py.test, notebooks ################################################################################ -cd $WORKSPACE/python/cudf +cd "$WORKSPACE/python/cudf" gpuci_logger "Python py.test for cuDF" -py.test -n 6 --cache-clear --basetemp=${WORKSPACE}/cudf-cuda-tmp --junitxml=${WORKSPACE}/junit-cudf.xml -v --cov-config=.coveragerc --cov=cudf --cov-report=xml:${WORKSPACE}/python/cudf/cudf-coverage.xml --cov-report term +py.test -n 6 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --junitxml="$WORKSPACE/junit-cudf.xml" -v --cov-config=.coveragerc --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-coverage.xml" --cov-report term -cd $WORKSPACE/python/dask_cudf +cd "$WORKSPACE/python/dask_cudf" gpuci_logger "Python py.test for dask-cudf" -py.test -n 6 --cache-clear --basetemp=${WORKSPACE}/dask-cudf-cuda-tmp --junitxml=${WORKSPACE}/junit-dask-cudf.xml -v --cov-config=.coveragerc --cov=dask_cudf --cov-report=xml:${WORKSPACE}/python/dask_cudf/dask-cudf-coverage.xml --cov-report term +py.test -n 6 --cache-clear --basetemp="$WORKSPACE/dask-cudf-cuda-tmp" --junitxml="$WORKSPACE/junit-dask-cudf.xml" -v --cov-config=.coveragerc --cov=dask_cudf --cov-report=xml:"$WORKSPACE/python/dask_cudf/dask-cudf-coverage.xml" --cov-report term -cd $WORKSPACE/python/custreamz +cd "$WORKSPACE/python/custreamz" gpuci_logger "Python py.test for cuStreamz" -py.test -n 6 --cache-clear --basetemp=${WORKSPACE}/custreamz-cuda-tmp --junitxml=${WORKSPACE}/junit-custreamz.xml -v --cov-config=.coveragerc --cov=custreamz --cov-report=xml:${WORKSPACE}/python/custreamz/custreamz-coverage.xml --cov-report term +py.test -n 6 --cache-clear --basetemp="$WORKSPACE/custreamz-cuda-tmp" --junitxml="$WORKSPACE/junit-custreamz.xml" -v --cov-config=.coveragerc --cov=custreamz --cov-report=xml:"$WORKSPACE/python/custreamz/custreamz-coverage.xml" --cov-report term gpuci_logger "Test notebooks" -${WORKSPACE}/ci/gpu/test-notebooks.sh 2>&1 | tee nbtest.log -python ${WORKSPACE}/ci/utils/nbtestlog2junitxml.py nbtest.log +"$WORKSPACE/ci/gpu/test-notebooks.sh" 2>&1 | tee nbtest.log +python "$WORKSPACE/ci/utils/nbtestlog2junitxml.py" nbtest.log if [ -n "${CODECOV_TOKEN}" ]; then codecov -t $CODECOV_TOKEN diff --git a/ci/gpu/test-notebooks.sh b/ci/gpu/test-notebooks.sh index ffa2e2a7214..1a5c2614000 100755 --- a/ci/gpu/test-notebooks.sh +++ b/ci/gpu/test-notebooks.sh @@ -1,8 +1,8 @@ #!/bin/bash -NOTEBOOKS_DIR=${WORKSPACE}/notebooks -NBTEST=${WORKSPACE}/ci/utils/nbtest.sh -LIBCUDF_KERNEL_CACHE_PATH=${WORKSPACE}/.jitcache +NOTEBOOKS_DIR="$WORKSPACE/notebooks" +NBTEST="$WORKSPACE/ci/utils/nbtest.sh" +LIBCUDF_KERNEL_CACHE_PATH="$WORKSPACE/.jitcache" cd ${NOTEBOOKS_DIR} TOPLEVEL_NB_FOLDERS=$(find . -name *.ipynb |cut -d'/' -f2|sort -u) diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index f7b9774c6fd..1b39f267c65 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -22,7 +22,7 @@ get_ipython().run_cell_magic=my_run_cell_magic NO_COLORS=--colors=NoColor EXITCODE=0 -NBTMPDIR=${WORKSPACE}/tmp +NBTMPDIR="$WORKSPACE/tmp" mkdir -p ${NBTMPDIR} for nb in $*; do diff --git a/java/ci/build-in-docker.sh b/java/ci/build-in-docker.sh index 25f36f04e48..e26ac455433 100755 --- a/java/ci/build-in-docker.sh +++ b/java/ci/build-in-docker.sh @@ -30,7 +30,7 @@ OUT=${OUT:-out} SIGN_FILE=$1 #Set absolute path for OUT_PATH -OUT_PATH=$WORKSPACE/$OUT +OUT_PATH="$WORKSPACE/$OUT" # set on Jenkins parameter echo "SIGN_FILE: $SIGN_FILE,\ @@ -52,26 +52,27 @@ export LIBCUDF_KERNEL_CACHE_PATH=/rapids export PATH=/usr/local/cmake-3.19.0-Linux-x86_64/bin:$PATH ###### Build libcudf ###### -rm -rf $WORKSPACE/cpp/build -mkdir -p $WORKSPACE/cpp/build -cd $WORKSPACE/cpp/build +rm -rf "$WORKSPACE/cpp/build" +mkdir -p "$WORKSPACE/cpp/build" +cd "$WORKSPACE/cpp/build" cmake .. -DUSE_NVTX=$ENABLE_NVTX -DCUDF_USE_ARROW_STATIC=ON -DBUILD_TESTS=$SKIP_CPP_TESTS -DPER_THREAD_DEFAULT_STREAM=$ENABLE_PTDS -DRMM_LOGGING_LEVEL=$RMM_LOGGING_LEVEL + make -j$PARALLEL_LEVEL make install DESTDIR=$INSTALL_PREFIX ###### Build cudf jar ###### -BUILD_ARG="-Dmaven.repo.local=$WORKSPACE/.m2 -DskipTests=$SKIP_JAVA_TESTS -DPER_THREAD_DEFAULT_STREAM=$ENABLE_PTDS -DRMM_LOGGING_LEVEL=$RMM_LOGGING_LEVEL -DUSE_GDS=$ENABLE_GDS" +BUILD_ARG="-Dmaven.repo.local=\"$WORKSPACE/.m2\" -DskipTests=$SKIP_JAVA_TESTS -DPER_THREAD_DEFAULT_STREAM=$ENABLE_PTDS -DRMM_LOGGING_LEVEL=$RMM_LOGGING_LEVEL -DUSE_GDS=$ENABLE_GDS" if [ "$SIGN_FILE" == true ]; then # Build javadoc and sources only when SIGN_FILE is true BUILD_ARG="$BUILD_ARG -Prelease" fi -if [ -f $WORKSPACE/java/ci/settings.xml ]; then +if [ -f "$WORKSPACE/java/ci/settings.xml" ]; then # Build with an internal settings.xml - BUILD_ARG="$BUILD_ARG -s $WORKSPACE/java/ci/settings.xml" + BUILD_ARG="$BUILD_ARG -s \"$WORKSPACE/java/ci/settings.xml\"" fi -cd $WORKSPACE/java +cd "$WORKSPACE/java" mvn -B clean package $BUILD_ARG ###### Sanity test: fail if static cudart found ######