From 8630e7cbc3d7ecb23fec5d5b837b15b9a999f323 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 10 Feb 2023 19:08:34 -0600 Subject: [PATCH] Remove gpuCI scripts. (#12712) Since migrating to GitHub Actions, the gpuCI scripts are no longer needed. This PR removes those outdated gpuCI scripts. Authors: - Bradley Dice (https://github.com/bdice) - AJ Schmidt (https://github.com/ajschmidt8) Approvers: - Ray Douglass (https://github.com/raydouglass) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cudf/pull/12712 --- README.md | 2 - ci/benchmark/build.sh | 196 ----------------- ci/checks/changelog.sh | 39 ---- ci/checks/style.sh | 23 -- ci/cpu/build.sh | 150 ------------- ci/cpu/prebuild.sh | 15 -- ci/cpu/upload.sh | 65 ------ ci/gpu/build.sh | 324 ---------------------------- ci/gpu/java.sh | 115 ---------- ci/gpu/test-notebooks.sh | 48 ----- ci/local/README.md | 57 ----- ci/local/build.sh | 146 ------------- ci/release/update-version.sh | 5 - ci/test_cpp.sh | 2 +- ci/test_python_cudf.sh | 1 - conda/recipes/dask-cudf/run_test.sh | 8 +- 16 files changed, 5 insertions(+), 1191 deletions(-) delete mode 100755 ci/benchmark/build.sh delete mode 100755 ci/checks/changelog.sh delete mode 100755 ci/checks/style.sh delete mode 100755 ci/cpu/build.sh delete mode 100755 ci/cpu/prebuild.sh delete mode 100755 ci/cpu/upload.sh delete mode 100755 ci/gpu/build.sh delete mode 100755 ci/gpu/java.sh delete mode 100755 ci/gpu/test-notebooks.sh delete mode 100644 ci/local/README.md delete mode 100755 ci/local/build.sh diff --git a/README.md b/README.md index 68c2d4f6276..36c1ff1d1fa 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ #
 cuDF - GPU DataFrames
-[![Build Status](https://gpuci.gpuopenanalytics.com/job/rapidsai/job/gpuci/job/cudf/job/branches/job/cudf-branch-pipeline/badge/icon)](https://gpuci.gpuopenanalytics.com/job/rapidsai/job/gpuci/job/cudf/job/branches/job/cudf-branch-pipeline/) - **NOTE:** For the latest stable [README.md](https://github.com/rapidsai/cudf/blob/main/README.md) ensure you are on the `main` branch. ## Resources diff --git a/ci/benchmark/build.sh b/ci/benchmark/build.sh deleted file mode 100755 index e221424d1cd..00000000000 --- a/ci/benchmark/build.sh +++ /dev/null @@ -1,196 +0,0 @@ -#!/bin/bash -# Copyright (c) 2020-2023, NVIDIA CORPORATION. -######################################### -# cuDF GPU build and test script for CI # -######################################### -set -e -NUMARGS=$# -ARGS=$* - -# Logger function for build status output -function logger() { - echo -e "\n>>>> $@\n" -} - -# Arg parsing function -function hasArg { - (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") -} - -# Set path and build parallel level -export PATH=/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=4 -export CUDA_REL=${CUDA_VERSION%.*} -export HOME="$WORKSPACE" - -# Parse git describe -cd "$WORKSPACE" -export GIT_DESCRIBE_TAG=`git describe --tags` -export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` - -# Set Benchmark Vars -export GBENCH_BENCHMARKS_DIR="$WORKSPACE/cpp/build/gbenchmarks/" - -# Set `LIBCUDF_KERNEL_CACHE_PATH` environment variable to $HOME/.jitify-cache because -# it's local to the container's virtual file system, and not shared with other CI jobs -# like `/tmp` is. -export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache" - -# Dask & Distributed option to install main(nightly) or `conda-forge` packages. -export INSTALL_DASK_MAIN=1 - -# Dask version to install when `INSTALL_DASK_MAIN=0` -export DASK_STABLE_VERSION="2023.1.1" - -function remove_libcudf_kernel_cache_dir { - EXITCODE=$? - logger "removing kernel cache dir: $LIBCUDF_KERNEL_CACHE_PATH" - rm -rf "$LIBCUDF_KERNEL_CACHE_PATH" || logger "could not rm -rf $LIBCUDF_KERNEL_CACHE_PATH" - exit $EXITCODE -} - -trap remove_libcudf_kernel_cache_dir EXIT - -mkdir -p "$LIBCUDF_KERNEL_CACHE_PATH" || logger "could not mkdir -p $LIBCUDF_KERNEL_CACHE_PATH" - -################################################################################ -# SETUP - Check environment -################################################################################ - -logger "Check environment..." -env - -logger "Check GPU usage..." -nvidia-smi - -logger "Activate conda env..." -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -# Enter dependencies to be shown in ASV tooltips. -CUDF_DEPS=(librmm) -LIBCUDF_DEPS=(librmm) - -conda install "rmm=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \ - "rapids-build-env=$MINOR_VERSION.*" \ - "rapids-notebook-env=$MINOR_VERSION.*" \ - rapids-pytest-benchmark - -# https://docs.rapids.ai/maintainers/depmgmt/ -# conda remove -f rapids-build-env rapids-notebook-env -# conda install "your-pkg=1.0.0" - -# Install the conda-forge or nightly version of dask and distributed -if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then - gpuci_logger "gpuci_mamba_retry install -c dask/label/dev 'dask/label/dev::dask' 'dask/label/dev::distributed'" - gpuci_mamba_retry install -c dask/label/dev "dask/label/dev::dask" "dask/label/dev::distributed" -else - gpuci_logger "gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall" - gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall -fi - -# Install the master version of streamz -logger "pip install git+https://github.com/python-streamz/streamz.git@master --upgrade --no-deps" -pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps - -logger "Check versions..." -python --version - -conda info -conda config --show-sources -conda list --show-channel-urls - -################################################################################ -# BUILD - Build libcudf, cuDF and dask_cudf from source -################################################################################ - -logger "Build libcudf..." -"$WORKSPACE/build.sh" clean libcudf cudf dask_cudf benchmarks tests --ptds - -################################################################################ -# BENCHMARK - Run and parse libcudf and cuDF benchmarks -################################################################################ - -logger "Running benchmarks..." - -#Download GBench results Parser -curl -L https://raw.githubusercontent.com/rapidsai/benchmark/main/parser/GBenchToASV.py --output GBenchToASV.py - -### -# Generate Metadata for dependencies -### - -# Concatenate dependency arrays, convert to JSON array, -# and remove duplicates. -X=("${CUDF_DEPS[@]}" "${LIBCUDF_DEPS[@]}") -DEPS=$(printf '%s\n' "${X[@]}" | jq -R . | jq -s 'unique') - -# Build object with k/v pairs of "dependency:version" -DEP_VER_DICT=$(jq -n '{}') -for DEP in $(echo "${DEPS}" | jq -r '.[]'); do - VER=$(conda list | grep "^${DEP}" | awk '{print $2"-"$3}') - DEP_VER_DICT=$(echo "${DEP_VER_DICT}" | jq -c --arg DEP "${DEP}" --arg VER "${VER}" '. + { ($DEP): $VER }') -done - -# Pass in an array of dependencies to get a dict of "dependency:version" -function getReqs() { - local DEPS_ARR=("$@") - local REQS="{}" - for DEP in "${DEPS_ARR[@]}"; do - VER=$(echo "${DEP_VER_DICT}" | jq -r --arg DEP "${DEP}" '.[$DEP]') - REQS=$(echo "${REQS}" | jq -c --arg DEP "${DEP}" --arg VER "${VER}" '. + { ($DEP): $VER }') - done - - echo "${REQS}" -} - -### -# Run LIBCUDF Benchmarks -### - -REQS=$(getReqs "${LIBCUDF_DEPS[@]}") - -mkdir -p "$WORKSPACE/tmp/benchmark" -touch "$WORKSPACE/tmp/benchmark/benchmarks.txt" -ls ${GBENCH_BENCHMARKS_DIR} > "$WORKSPACE/tmp/benchmark/benchmarks.txt" - -#Disable error aborting while tests run, failed tests will not generate data -logger "Running libcudf GBenchmarks..." -cd ${GBENCH_BENCHMARKS_DIR} -set +e -while read BENCH; -do - nvidia-smi - ./${BENCH} --benchmark_out=${BENCH}.json --benchmark_out_format=json - EXITCODE=$? - if [[ ${EXITCODE} != 0 ]]; then - rm ./${BENCH}.json - JOBEXITCODE=1 - fi -done < "$WORKSPACE/tmp/benchmark/benchmarks.txt" -set -e - -rm "$WORKSPACE/tmp/benchmark/benchmarks.txt" -cd "$WORKSPACE" -mv ${GBENCH_BENCHMARKS_DIR}/*.json "$WORKSPACE/tmp/benchmark/" -python GBenchToASV.py -d "$WORKSPACE/tmp/benchmark/" -t ${S3_ASV_DIR} -n libcudf -b branch-${MINOR_VERSION} -r "${REQS}" - -### -# Run Python Benchmarks -### - -#REQS=$(getReqs "${CUDF_DEPS[@]}") - -#BENCHMARK_META=$(jq -n \ -# --arg NODE "${NODE_NAME}" \ -# --arg BRANCH "branch-${MINOR_VERSION}" \ -# --argjson REQS "${REQS}" ' -# { -# "machineName": $NODE, -# "commitBranch": $BRANCH, -# "requirements": $REQS -# } -#') - -#echo "Benchmark meta:" -#echo "${BENCHMARK_META}" | jq "." diff --git a/ci/checks/changelog.sh b/ci/checks/changelog.sh deleted file mode 100755 index 0dfcf27298e..00000000000 --- a/ci/checks/changelog.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018, NVIDIA CORPORATION. -######################### -# cuDF CHANGELOG Tester # -######################### - -# Checkout main for comparison -git checkout --force --quiet main - -# Switch back to tip of PR branch -git checkout --force --quiet current-pr-branch - -# Ignore errors during searching -set +e - -# Get list of modified files between main and PR branch -CHANGELOG=`git diff --name-only main...current-pr-branch | grep CHANGELOG.md` -# Check if CHANGELOG has PR ID -PRNUM=`cat CHANGELOG.md | grep "$PR_ID"` -RETVAL=0 - -# Return status of check result -if [ "$CHANGELOG" != "" -a "$PRNUM" != "" ] ; then - echo -e "\n\n>>>> PASSED: CHANGELOG.md has been updated with current PR information.\n\nPlease ensure the update meets the following criteria.\n" -else - echo -e "\n\n>>>> FAILED: CHANGELOG.md has not been updated!\n\nPlease add a line describing this PR to CHANGELOG.md in the repository root directory. The line should meet the following criteria.\n" - RETVAL=1 -fi - -cat << EOF - It should be placed under the section for the appropriate release. - It should be placed under "New Features", "Improvements", or "Bug Fixes" as appropriate. - It should be formatted as '- PR # ' - Example format for #491 '- PR #491 Add CI test script to check for updates to CHANGELOG.md in PRs' - - -EOF - -exit $RETVAL diff --git a/ci/checks/style.sh b/ci/checks/style.sh deleted file mode 100755 index d32d88f5574..00000000000 --- a/ci/checks/style.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. -##################### -# cuDF Style Tester # -##################### - -# Ignore errors and set path -set +e -PATH=/conda/bin:$PATH -LC_ALL=C.UTF-8 -LANG=C.UTF-8 - -# Activate common conda env -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -FORMAT_FILE_URL=https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.04/cmake-format-rapids-cmake.json -export RAPIDS_CMAKE_FORMAT_FILE=/tmp/rapids_cmake_ci/cmake-formats-rapids-cmake.json -mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE}) -wget -O ${RAPIDS_CMAKE_FORMAT_FILE} ${FORMAT_FILE_URL} - -# Run pre-commit checks -pre-commit run --hook-stage manual --all-files --show-diff-on-failure diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh deleted file mode 100755 index 3aa00ff7de9..00000000000 --- a/ci/cpu/build.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2023, NVIDIA CORPORATION. -############################################## -# cuDF CPU conda build script for CI # -############################################## -set -e - -# Set path and build parallel level -# FIXME: PATH variable shouldn't be necessary. -# This should be removed once we either stop using the `remote-docker-plugin` -# or the following issue is addressed: https://github.com/gpuopenanalytics/remote-docker-plugin/issues/47 -export PATH=/usr/local/gcc9/bin:/opt/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} - -# Set home to the job's workspace -export HOME="$WORKSPACE" - -# Determine CUDA release version -export CUDA_REL=${CUDA_VERSION%.*} - -# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts) -export GPUCI_CONDA_RETRY_MAX=1 -export GPUCI_CONDA_RETRY_SLEEP=30 - -# Workaround to keep Jenkins builds working -# until we migrate fully to GitHub Actions -export RAPIDS_CUDA_VERSION="${CUDA}" -export SCCACHE_BUCKET=rapids-sccache -export SCCACHE_REGION=us-west-2 -export SCCACHE_IDLE_TIMEOUT=32768 - -# Use Ninja to build, setup Conda Build Dir -export CMAKE_GENERATOR="Ninja" -export CONDA_BLD_DIR="$WORKSPACE/.conda-bld" - -# Whether to keep `dask/label/dev` channel in the env. If INSTALL_DASK_MAIN=0, -# `dask/label/dev` channel is removed. -export INSTALL_DASK_MAIN=1 - -# Switch to project root; also root of repo checkout -cd "$WORKSPACE" - -# If nightly build, append current YYMMDD to version -if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then - export VERSION_SUFFIX=`date +%y%m%d` -fi - -################################################################################ -# SETUP - Check environment -################################################################################ - -gpuci_logger "Check environment variables" -env - -gpuci_logger "Activate conda env" -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -# Remove `rapidsai-nightly` & `dask/label/dev` channel if we are building main branch -if [ "$SOURCE_BRANCH" = "main" ]; then - conda config --system --remove channels rapidsai-nightly - conda config --system --remove channels dask/label/dev -elif [[ "${INSTALL_DASK_MAIN}" == 0 ]]; then - # Remove `dask/label/dev` channel if INSTALL_DASK_MAIN=0 - conda config --system --remove channels dask/label/dev -fi - -gpuci_logger "Check compiler versions" -python --version - -gpuci_logger "Check conda environment" -conda info -conda config --show-sources -conda list --show-channel-urls - -# FIX Added to deal with Anancoda SSL verification issues during conda builds -conda config --set ssl_verify False - -# TODO: Move boa install to gpuci/rapidsai -gpuci_mamba_retry install boa -################################################################################ -# BUILD - Conda package builds -################################################################################ - -if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then - CONDA_BUILD_ARGS="" - CONDA_CHANNEL="" -else - CONDA_BUILD_ARGS="--dirty --no-remove-work-dir" - CONDA_CHANNEL="-c $WORKSPACE/ci/artifacts/cudf/cpu/.conda-bld/" -fi - -if [ "$BUILD_LIBCUDF" == '1' ]; then - gpuci_logger "Build conda pkg for libcudf" - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcudf $CONDA_BUILD_ARGS - - # BUILD_LIBCUDF == 1 means this job is being run on the cpu_build jobs - # that is where we must also build the strings_udf package - mkdir -p ${CONDA_BLD_DIR}/strings_udf/work - STRINGS_UDF_BUILD_DIR=${CONDA_BLD_DIR}/strings_udf/work - gpuci_logger "Build conda pkg for cudf (python 3.8), for strings_udf" - gpuci_conda_retry mambabuild --no-build-id --croot ${STRINGS_UDF_BUILD_DIR} -c ${CONDA_BLD_DIR} conda/recipes/cudf ${CONDA_BUILD_ARGS} --python=3.8 - gpuci_logger "Build conda pkg for cudf (python 3.9), for strings_udf" - gpuci_conda_retry mambabuild --no-build-id --croot ${STRINGS_UDF_BUILD_DIR} -c ${CONDA_BLD_DIR} conda/recipes/cudf ${CONDA_BUILD_ARGS} --python=3.9 - - gpuci_logger "Build conda pkg for strings_udf (python 3.8)" - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} -c ${STRINGS_UDF_BUILD_DIR} -c ${CONDA_BLD_DIR} conda/recipes/strings_udf $CONDA_BUILD_ARGS --python=3.8 - gpuci_logger "Build conda pkg for strings_udf (python 3.9)" - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} -c ${STRINGS_UDF_BUILD_DIR} -c ${CONDA_BLD_DIR} conda/recipes/strings_udf $CONDA_BUILD_ARGS --python=3.9 - - mkdir -p ${CONDA_BLD_DIR}/libcudf/work - cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/libcudf/work - gpuci_logger "sccache stats" - sccache --show-stats - - # Copy libcudf build metrics results - LIBCUDF_BUILD_DIR=$CONDA_BLD_DIR/libcudf/work/cpp/build - echo "Checking for build metrics log $LIBCUDF_BUILD_DIR/ninja_log.html" - if [[ -f "$LIBCUDF_BUILD_DIR/ninja_log.html" ]]; then - gpuci_logger "Copying build metrics results" - mkdir -p "$WORKSPACE/build-metrics" - cp "$LIBCUDF_BUILD_DIR/ninja_log.html" "$WORKSPACE/build-metrics/BuildMetrics.html" - cp "$LIBCUDF_BUILD_DIR/ninja.log" "$WORKSPACE/build-metrics/ninja.log" - fi -fi - -if [ "$BUILD_CUDF" == '1' ]; then - gpuci_logger "Build conda pkg for cudf" - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/cudf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL - - gpuci_logger "Build conda pkg for dask-cudf" - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/dask-cudf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL - - gpuci_logger "Build conda pkg for cudf_kafka" - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/cudf_kafka --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL - - gpuci_logger "Build conda pkg for custreamz" - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/custreamz --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL - - gpuci_logger "Build conda pkg for strings_udf" - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/strings_udf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL - -fi -################################################################################ -# UPLOAD - Conda packages -################################################################################ - -# Uploads disabled due to new GH Actions implementation -# gpuci_logger "Upload conda pkgs" -# source ci/cpu/upload.sh diff --git a/ci/cpu/prebuild.sh b/ci/cpu/prebuild.sh deleted file mode 100755 index 32589042f7f..00000000000 --- a/ci/cpu/prebuild.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -set -e - -#Always upload cudf packages -export UPLOAD_CUDF=1 -export UPLOAD_LIBCUDF=1 -export UPLOAD_CUDF_KAFKA=1 - -if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then - #If project flash is not activate, always build both - export BUILD_LIBCUDF=1 - export BUILD_CUDF=1 -fi diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh deleted file mode 100755 index 82c58673605..00000000000 --- a/ci/cpu/upload.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. -# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh -# Copyright (c) 2020-2022, NVIDIA CORPORATION. - -set -e - -# Setup 'gpuci_retry' for upload retries (results in 4 total attempts) -export GPUCI_RETRY_MAX=3 -export GPUCI_RETRY_SLEEP=30 - -# Set default label options if they are not defined elsewhere -export LABEL_OPTION=${LABEL_OPTION:-"--label main"} - -# Skip uploads unless BUILD_MODE == "branch" -if [ "${BUILD_MODE}" != "branch" ]; then - echo "Skipping upload" - return 0 -fi - -# Skip uploads if there is no upload key -if [ -z "$MY_UPLOAD_KEY" ]; then - echo "No upload key" - return 0 -fi - -################################################################################ -# UPLOAD - Conda packages -################################################################################ - -gpuci_logger "Starting conda uploads" -if [[ "$BUILD_LIBCUDF" == "1" && "$UPLOAD_LIBCUDF" == "1" ]]; then - export LIBCUDF_FILES=$(conda build --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/libcudf --output) - LIBCUDF_FILES=$(echo "$LIBCUDF_FILES" | sed 's/.*libcudf-example.*//') # skip libcudf-example pkg upload - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing --no-progress $LIBCUDF_FILES -fi - -if [[ "$BUILD_CUDF" == "1" && "$UPLOAD_CUDF" == "1" ]]; then - export CUDF_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/cudf --python=$PYTHON --output) - test -e ${CUDF_FILE} - echo "Upload cudf: ${CUDF_FILE}" - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUDF_FILE} --no-progress - - export STRINGS_UDF_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/strings_udf --python=$PYTHON --output -c "${CONDA_BLD_DIR}") - test -e ${STRINGS_UDF_FILE} - echo "Upload strings_udf: ${STRINGS_UDF_FILE}" - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${STRINGS_UDF_FILE} --no-progress - - export DASK_CUDF_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/dask-cudf --python=$PYTHON --output) - test -e ${DASK_CUDF_FILE} - echo "Upload dask-cudf: ${DASK_CUDF_FILE}" - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${DASK_CUDF_FILE} --no-progress - - export CUSTREAMZ_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/custreamz --python=$PYTHON --output) - test -e ${CUSTREAMZ_FILE} - echo "Upload custreamz: ${CUSTREAMZ_FILE}" - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUSTREAMZ_FILE} --no-progress -fi - -if [[ "$BUILD_CUDF" == "1" && "$UPLOAD_CUDF_KAFKA" == "1" ]]; then - export CUDF_KAFKA_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/cudf_kafka --python=$PYTHON --output) - test -e ${CUDF_KAFKA_FILE} - echo "Upload cudf_kafka: ${CUDF_KAFKA_FILE}" - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUDF_KAFKA_FILE} --no-progress -fi diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh deleted file mode 100755 index ff40fdf6c9f..00000000000 --- a/ci/gpu/build.sh +++ /dev/null @@ -1,324 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2023, NVIDIA CORPORATION. -############################################## -# cuDF GPU build and test script for CI # -############################################## -set -e -NUMARGS=$# -ARGS=$* - -# Arg parsing function -function hasArg { - (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") -} - -# Set path and build parallel level -export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} - -# Set home to the job's workspace -export HOME="$WORKSPACE" - -# Switch to project root; also root of repo checkout -cd "$WORKSPACE" - -# Determine CUDA release version -export CUDA_REL=${CUDA_VERSION%.*} -export CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/cudf/cpu/.conda-bld/" - -# Workaround to keep Jenkins builds working -# until we migrate fully to GitHub Actions -export RAPIDS_CUDA_VERSION="${CUDA}" -export SCCACHE_BUCKET=rapids-sccache -export SCCACHE_REGION=us-west-2 -export SCCACHE_IDLE_TIMEOUT=32768 - -# Parse git describe -export GIT_DESCRIBE_TAG=`git describe --tags` -export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` -unset GIT_DESCRIBE_TAG - -# Dask & Distributed option to install main(nightly) or `conda-forge` packages. -export INSTALL_DASK_MAIN=1 - -# Dask version to install when `INSTALL_DASK_MAIN=0` -export DASK_STABLE_VERSION="2023.1.1" - -# ucx-py version -export UCX_PY_VERSION='0.31.*' - -################################################################################ -# TRAP - Setup trap for removing jitify cache -################################################################################ - -# Set `LIBCUDF_KERNEL_CACHE_PATH` environment variable to $HOME/.jitify-cache -# because it's local to the container's virtual file system, and not shared with -# other CI jobs like `/tmp` is -export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache" - -function remove_libcudf_kernel_cache_dir { - EXITCODE=$? - gpuci_logger "TRAP: Removing kernel cache dir: $LIBCUDF_KERNEL_CACHE_PATH" - rm -rf "$LIBCUDF_KERNEL_CACHE_PATH" \ - || gpuci_logger "[ERROR] TRAP: Could not rm -rf $LIBCUDF_KERNEL_CACHE_PATH" - exit $EXITCODE -} - -# Set trap to run on exit -gpuci_logger "TRAP: Set trap to remove jitify cache on exit" -trap remove_libcudf_kernel_cache_dir EXIT - -mkdir -p "$LIBCUDF_KERNEL_CACHE_PATH" \ - || gpuci_logger "[ERROR] TRAP: Could not mkdir -p $LIBCUDF_KERNEL_CACHE_PATH" - -################################################################################ -# SETUP - Check environment -################################################################################ - -gpuci_logger "Check environment variables" -env - -gpuci_logger "Check GPU usage" -nvidia-smi - -gpuci_logger "Activate conda env" -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -# Remove `dask/label/dev` channel if INSTALL_DASK_MAIN=0 -if [ "$SOURCE_BRANCH" != "main" ] && [[ "${INSTALL_DASK_MAIN}" == 0 ]]; then - conda config --system --remove channels dask/label/dev - gpuci_mamba_retry install conda-forge::dask==$DASK_STABLE_VERSION conda-forge::distributed==$DASK_STABLE_VERSION conda-forge::dask-core==$DASK_STABLE_VERSION --force-reinstall -fi - -gpuci_logger "Check conda environment" -conda info -conda config --show-sources -conda list --show-channel-urls -gpuci_logger "Check compiler versions" -python --version - -function install_dask { - # Install the conda-forge or nightly version of dask and distributed - gpuci_logger "Install the conda-forge or nightly version of dask and distributed" - set -x - if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then - gpuci_logger "gpuci_mamba_retry install -c dask/label/dev 'dask/label/dev::dask' 'dask/label/dev::distributed'" - gpuci_mamba_retry install -c dask/label/dev "dask/label/dev::dask" "dask/label/dev::distributed" - conda list - else - gpuci_logger "gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall" - gpuci_mamba_retry install conda-forge::dask==$DASK_STABLE_VERSION conda-forge::distributed==$DASK_STABLE_VERSION conda-forge::dask-core==$DASK_STABLE_VERSION --force-reinstall - fi - # Install the main version of streamz - gpuci_logger "Install the main version of streamz" - # Need to uninstall streamz that is already in the env. - pip uninstall -y streamz - pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps - set +x -} - -install_dask - -if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then - - gpuci_logger "Install dependencies" - gpuci_mamba_retry install -y \ - "cudatoolkit=$CUDA_REL" \ - "rapids-build-env=$MINOR_VERSION.*" \ - "rapids-notebook-env=$MINOR_VERSION.*" \ - "dask-cuda=${MINOR_VERSION}" \ - "rmm=$MINOR_VERSION.*" \ - "ucx-py=${UCX_PY_VERSION}" - - # https://docs.rapids.ai/maintainers/depmgmt/ - # gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env - # gpuci_mamba_retry install -y "your-pkg=1.0.0" - - ################################################################################ - # BUILD - Build libcudf, cuDF, libcudf_kafka, dask_cudf, and strings_udf from source - ################################################################################ - - gpuci_logger "Build from source" - "$WORKSPACE/build.sh" clean libcudf cudf dask_cudf libcudf_kafka cudf_kafka strings_udf benchmarks tests --ptds - - ################################################################################ - # TEST - Run GoogleTest - ################################################################################ - - set +e -Eo pipefail - EXITCODE=0 - trap "EXITCODE=1" ERR - - - if hasArg --skip-tests; then - gpuci_logger "Skipping Tests" - exit 0 - else - gpuci_logger "Check GPU usage" - nvidia-smi - - gpuci_logger "GoogleTests" - set -x - cd "$WORKSPACE/cpp/build" - - for gt in "$WORKSPACE/cpp/build/gtests/"* ; do - test_name=$(basename ${gt}) - echo "Running GoogleTest $test_name" - ${gt} --gtest_output=xml:"$WORKSPACE/test-results/" - done - fi -else - #Project Flash - - if hasArg --skip-tests; then - gpuci_logger "Skipping Tests" - exit 0 - fi - - gpuci_logger "Check GPU usage" - nvidia-smi - - gpuci_logger "Installing libcudf, libcudf_kafka and libcudf-tests" - gpuci_mamba_retry install -y -c ${CONDA_ARTIFACT_PATH} libcudf libcudf_kafka libcudf-tests - - # TODO: Move boa install to gpuci/rapidsai - gpuci_mamba_retry install boa - gpuci_logger "Building cudf, dask-cudf, cudf_kafka and custreamz" - export CONDA_BLD_DIR="$WORKSPACE/.conda-bld" - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/cudf --python=$PYTHON -c ${CONDA_ARTIFACT_PATH} - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/dask-cudf --python=$PYTHON -c ${CONDA_ARTIFACT_PATH} - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/cudf_kafka --python=$PYTHON -c ${CONDA_ARTIFACT_PATH} - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/custreamz --python=$PYTHON -c ${CONDA_ARTIFACT_PATH} - - # the CUDA component of strings_udf must be built on cuda 11.5 just like libcudf - # but because there is no separate python package, we must also build the python on the 11.5 jobs - # this means that at this point (on the GPU test jobs) the whole package is already built and has been - # copied by CI from the upstream 11.5 jobs into $CONDA_ARTIFACT_PATH - gpuci_logger "Installing cudf, dask-cudf, cudf_kafka, and custreamz" - gpuci_mamba_retry install cudf dask-cudf cudf_kafka custreamz -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" - - gpuci_logger "Check current conda environment" - conda list --show-channel-urls - - gpuci_logger "GoogleTests" - - # Set up library for finding incorrect default stream usage. - cd "$WORKSPACE/cpp/tests/utilities/identify_stream_usage/" - mkdir build && cd build && cmake .. -GNinja && ninja && ninja test - STREAM_IDENTIFY_LIB="$WORKSPACE/cpp/tests/utilities/identify_stream_usage/build/libidentify_stream_usage.so" - - # Run libcudf and libcudf_kafka gtests from libcudf-tests package - for gt in "$CONDA_PREFIX/bin/gtests/libcudf"*/* ; do - test_name=$(basename ${gt}) - - echo "Running GoogleTest $test_name" - if [[ ${test_name} == "SPAN_TEST" ]]; then - # This one test is specifically designed to test using a thrust device - # vector, so we expect and allow it to include default stream usage. - gtest_filter="SpanTest.CanConstructFromDeviceContainers" - GTEST_CUDF_STREAM_MODE="custom" LD_PRELOAD=${STREAM_IDENTIFY_LIB} ${gt} --gtest_output=xml:"$WORKSPACE/test-results/" --gtest_filter="-${gtest_filter}" - ${gt} --gtest_output=xml:"$WORKSPACE/test-results/" --gtest_filter="${gtest_filter}" - else - GTEST_CUDF_STREAM_MODE="custom" LD_PRELOAD=${STREAM_IDENTIFY_LIB} ${gt} --gtest_output=xml:"$WORKSPACE/test-results/" - fi - done - - export LIB_BUILD_DIR="$WORKSPACE/ci/artifacts/cudf/cpu/libcudf_work/cpp/build" - # Copy libcudf build time results - echo "Checking for build time log $LIB_BUILD_DIR/ninja_log.xml" - if [[ -f "$LIB_BUILD_DIR/ninja_log.xml" ]]; then - gpuci_logger "Copying build time results" - cp "$LIB_BUILD_DIR/ninja_log.xml" "$WORKSPACE/test-results/buildtimes-junit.xml" - fi - - ################################################################################ - # MEMCHECK - Run compute-sanitizer on GoogleTest (only in nightly builds) - ################################################################################ - if [[ "$BUILD_MODE" == "branch" && "$BUILD_TYPE" == "gpu" ]]; then - if [[ "$COMPUTE_SANITIZER_ENABLE" == "true" ]]; then - gpuci_logger "Memcheck on GoogleTests with rmm_mode=cuda" - export GTEST_CUDF_RMM_MODE=cuda - COMPUTE_SANITIZER_CMD="compute-sanitizer --tool memcheck" - mkdir -p "$WORKSPACE/test-results/" - for gt in "$CONDA_PREFIX/bin/gtests/libcudf"*/* ; do - test_name=$(basename ${gt}) - if [[ "$test_name" == "ERROR_TEST" ]]; then - continue - fi - echo "Running GoogleTest $test_name" - ${COMPUTE_SANITIZER_CMD} ${gt} | tee "$WORKSPACE/test-results/${test_name}.cs.log" - done - unset GTEST_CUDF_RMM_MODE - # test-results/*.cs.log are processed in gpuci - fi - fi -fi - -# Both regular and Project Flash proceed here - -# set environment variable for numpy 1.16 -# will be enabled for later versions by default -np_ver=$(python -c "import numpy; print('.'.join(numpy.__version__.split('.')[:-1]))") -if [ "$np_ver" == "1.16" ];then - export NUMPY_EXPERIMENTAL_ARRAY_FUNCTION=1 -fi - -################################################################################ -# TEST - Run py.test, notebooks -################################################################################ - -cd "$WORKSPACE/python/cudf/cudf" -# It is essential to cd into $WORKSPACE/python/cudf/cudf as `pytest-xdist` + `coverage` seem to work only at this directory level. -gpuci_logger "Check conda packages" -conda list -gpuci_logger "Python py.test for cuDF" -py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --ignore="$WORKSPACE/python/cudf/cudf/benchmarks" --junitxml="$WORKSPACE/junit-cudf.xml" -v --cov-config="$WORKSPACE/python/cudf/.coveragerc" --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-coverage.xml" --cov-report term --dist=loadscope tests - -gpuci_logger "Python py.tests for cuDF with spilling (CUDF_SPILL_DEVICE_LIMIT=1)" -# Due to time concerns, we only run tests marked "spilling" -CUDF_SPILL=on CUDF_SPILL_DEVICE_LIMIT=1 py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --ignore="$WORKSPACE/python/cudf/cudf/benchmarks" -v --cov-config="$WORKSPACE/python/cudf/.coveragerc" --cov-append --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-coverage.xml" --cov-report term --dist=loadscope -m spilling tests - -cd "$WORKSPACE/python/dask_cudf" -gpuci_logger "Python py.test for dask-cudf" -py.test -n 8 --cache-clear --basetemp="$WORKSPACE/dask-cudf-cuda-tmp" --junitxml="$WORKSPACE/junit-dask-cudf.xml" -v --cov-config=.coveragerc --cov=dask_cudf --cov-report=xml:"$WORKSPACE/python/dask_cudf/dask-cudf-coverage.xml" --cov-report term dask_cudf - -cd "$WORKSPACE/python/custreamz" -gpuci_logger "Python py.test for cuStreamz" -py.test -n 8 --cache-clear --basetemp="$WORKSPACE/custreamz-cuda-tmp" --junitxml="$WORKSPACE/junit-custreamz.xml" -v --cov-config=.coveragerc --cov=custreamz --cov-report=xml:"$WORKSPACE/python/custreamz/custreamz-coverage.xml" --cov-report term custreamz - - -# only install strings_udf after cuDF is finished testing without its presence -gpuci_logger "Installing strings_udf" -gpuci_mamba_retry install strings_udf -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" - -cd "$WORKSPACE/python/strings_udf/strings_udf" -gpuci_logger "Python py.test for strings_udf" -py.test -n 8 --cache-clear --basetemp="$WORKSPACE/strings-udf-cuda-tmp" --junitxml="$WORKSPACE/junit-strings-udf.xml" -v --cov-config=.coveragerc --cov=strings_udf --cov-report=xml:"$WORKSPACE/python/strings_udf/strings-udf-coverage.xml" --cov-report term tests - -# retest cuDF UDFs -cd "$WORKSPACE/python/cudf/cudf" -gpuci_logger "Python py.test retest cuDF UDFs" -py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-strings-udf-tmp" --ignore="$WORKSPACE/python/cudf/cudf/benchmarks" --junitxml="$WORKSPACE/junit-cudf-strings-udf.xml" -v --cov-config="$WORKSPACE/python/cudf/.coveragerc" --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-strings-udf-coverage.xml" --cov-report term --dist=loadscope tests/test_udf_masked_ops.py - - -# Run benchmarks with both cudf and pandas to ensure compatibility is maintained. -# Benchmarks are run in DEBUG_ONLY mode, meaning that only small data sizes are used. -# Therefore, these runs only verify that benchmarks are valid. -# They do not generate meaningful performance measurements. -cd "$WORKSPACE/python/cudf" -gpuci_logger "Python pytest for cuDF benchmarks" -CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" -v --dist=loadscope benchmarks - -gpuci_logger "Python pytest for cuDF benchmarks using pandas" -CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" -v --dist=loadscope benchmarks - -gpuci_logger "Test notebooks" -"$WORKSPACE/ci/gpu/test-notebooks.sh" 2>&1 | tee nbtest.log -python "$WORKSPACE/ci/utils/nbtestlog2junitxml.py" nbtest.log - -if [ -n "${CODECOV_TOKEN}" ]; then - codecov -t $CODECOV_TOKEN -fi - -return ${EXITCODE} diff --git a/ci/gpu/java.sh b/ci/gpu/java.sh deleted file mode 100755 index 2db9cd57eb8..00000000000 --- a/ci/gpu/java.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. -############################################## -# cuDF GPU build and test script for CI # -############################################## -set -e -NUMARGS=$# -ARGS=$* - -# Arg parsing function -function hasArg { - (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") -} - -# Set path and build parallel level -export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} - -# Set home to the job's workspace -export HOME="$WORKSPACE" - -# Switch to project root; also root of repo checkout -cd "$WORKSPACE" - -# Determine CUDA release version -export CUDA_REL=${CUDA_VERSION%.*} -export CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/cudf/cpu/.conda-bld/" - -# Parse git describe -export GIT_DESCRIBE_TAG=`git describe --tags` -export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` - -################################################################################ -# TRAP - Setup trap for removing jitify cache -################################################################################ - -# Set `LIBCUDF_KERNEL_CACHE_PATH` environment variable to $HOME/.jitify-cache -# because it's local to the container's virtual file system, and not shared with -# other CI jobs like `/tmp` is -export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache" - -function remove_libcudf_kernel_cache_dir { - EXITCODE=$? - gpuci_logger "TRAP: Removing kernel cache dir: $LIBCUDF_KERNEL_CACHE_PATH" - rm -rf "$LIBCUDF_KERNEL_CACHE_PATH" \ - || gpuci_logger "[ERROR] TRAP: Could not rm -rf $LIBCUDF_KERNEL_CACHE_PATH" - exit $EXITCODE -} - -# Set trap to run on exit -gpuci_logger "TRAP: Set trap to remove jitify cache on exit" -trap remove_libcudf_kernel_cache_dir EXIT - -mkdir -p "$LIBCUDF_KERNEL_CACHE_PATH" \ - || gpuci_logger "[ERROR] TRAP: Could not mkdir -p $LIBCUDF_KERNEL_CACHE_PATH" - -################################################################################ -# SETUP - Check environment -################################################################################ - -gpuci_logger "Check environment variables" -env - -gpuci_logger "Check GPU usage" -nvidia-smi - -gpuci_logger "Activate conda env" -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -gpuci_logger "Check conda environment" -conda info -conda config --show-sources -conda list --show-channel-urls - -gpuci_logger "Install dependencies" -gpuci_mamba_retry install -y \ - "cudatoolkit=$CUDA_REL" \ - "rapids-build-env=$MINOR_VERSION.*" \ - "rmm=$MINOR_VERSION.*" \ - "openjdk=8.*" \ - "maven" -# "mamba install openjdk" adds an activation script to set JAVA_HOME but this is -# not triggered on installation. Re-activating the conda environment will set -# this environment variable so that CMake can find JNI. -conda activate rapids - -# https://docs.rapids.ai/maintainers/depmgmt/ -# gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env -# gpuci_mamba_retry install -y "your-pkg=1.0.0" - -gpuci_logger "Check conda environment" -conda info -conda config --show-sources -conda list --show-channel-urls - -################################################################################ -# INSTALL - Install libcudf artifacts -################################################################################ - -gpuci_logger "Installing libcudf" -gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} libcudf - -################################################################################ -# TEST - Run java tests -################################################################################ - -gpuci_logger "Check GPU usage" -nvidia-smi - -gpuci_logger "Running Java Tests" -cd ${WORKSPACE}/java -mvn test -B -DCUDF_JNI_ARROW_STATIC=OFF - -return ${EXITCODE} diff --git a/ci/gpu/test-notebooks.sh b/ci/gpu/test-notebooks.sh deleted file mode 100755 index 36d093d0d28..00000000000 --- a/ci/gpu/test-notebooks.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# Copyright (c) 2020-2022, NVIDIA CORPORATION. - -NOTEBOOKS_DIR="$WORKSPACE/notebooks" -NBTEST="$WORKSPACE/ci/utils/nbtest.sh" -LIBCUDF_KERNEL_CACHE_PATH="$WORKSPACE/.jitcache" - -cd ${NOTEBOOKS_DIR} -TOPLEVEL_NB_FOLDERS=$(find . -name *.ipynb |cut -d'/' -f2|sort -u) - -# Add notebooks that should be skipped here -# (space-separated list of filenames without paths) - -SKIPNBS="" - -## Check env -env - -EXITCODE=0 - -# Always run nbtest in all TOPLEVEL_NB_FOLDERS, set EXITCODE to failure -# if any run fails - -cd ${NOTEBOOKS_DIR} -for nb in $(find . -name "*.ipynb"); do - nbBasename=$(basename ${nb}) - # Skip all NBs that use dask (in the code or even in their name) - if ((echo ${nb}|grep -qi dask) || \ - (grep -q dask ${nb})); then - echo "--------------------------------------------------------------------------------" - echo "SKIPPING: ${nb} (suspected Dask usage, not currently automatable)" - echo "--------------------------------------------------------------------------------" - elif (echo " ${SKIPNBS} " | grep -q " ${nbBasename} "); then - echo "--------------------------------------------------------------------------------" - echo "SKIPPING: ${nb} (listed in skip list)" - echo "--------------------------------------------------------------------------------" - else - nvidia-smi - ${NBTEST} ${nbBasename} - EXITCODE=$((EXITCODE | $?)) - rm -rf ${LIBCUDF_KERNEL_CACHE_PATH}/* - fi -done - - -nvidia-smi - -exit ${EXITCODE} diff --git a/ci/local/README.md b/ci/local/README.md deleted file mode 100644 index 7754bcaf647..00000000000 --- a/ci/local/README.md +++ /dev/null @@ -1,57 +0,0 @@ -## Purpose - -This script is designed for developer and contributor use. This tool mimics the actions of gpuCI on your local machine. This allows you to test and even debug your code inside a gpuCI base container before pushing your code as a GitHub commit. -The script can be helpful in locally triaging and debugging RAPIDS continuous integration failures. - -## Requirements - -``` -nvidia-docker -``` - -## Usage - -``` -bash build.sh [-h] [-H] [-s] [-r ] [-i ] -Build and test your local repository using a base gpuCI Docker image - -where: - -H Show this help text - -r Path to repository (defaults to working directory) - -i Use Docker image (default is gpuci/rapidsai:${NIGHTLY_VERSION}-cuda11.5-devel-ubuntu20.04-py3.8) - -s Skip building and testing and start an interactive shell in a container of the Docker image -``` - -Example Usage: -`bash build.sh -r ~/rapids/cudf -i gpuci/rapidsai:22.02-cuda11.5-devel-ubuntu20.04-py3.8` - -For a full list of available gpuCI docker images, visit our [DockerHub](https://hub.docker.com/r/gpuci/rapidsai/tags) page. - -Style Check: -```bash -$ bash ci/local/build.sh -r ~/rapids/cudf -s -$ source activate rapids # Activate gpuCI conda environment -$ cd rapids -$ flake8 python -``` - -## Information - -There are some caveats to be aware of when using this script, especially if you plan on developing from within the container itself. - - -### Docker Image Build Repository - -The docker image will generate build artifacts in a folder on your machine located in the `root` directory of the repository you passed to the script. For the above example, the directory is named `~/rapids/cudf/build_rapidsai_cuda11.5-ubuntu20.04-py3.8/`. Feel free to remove this directory after the script is finished. - -*Note*: The script *will not* override your local build repository. Your local environment stays in tact. - - -### Where The User is Dumped - -The script will build your repository and run all tests. If any tests fail, it dumps the user into the docker container itself to allow you to debug from within the container. If all the tests pass as expected the container exits and is automatically removed. Remember to exit the container if tests fail and you do not wish to debug within the container itself. - - -### Container File Structure - -Your repository will be located in the `/rapids/` folder of the container. This folder is volume mounted from the local machine. Any changes to the code in this repository are replicated onto the local machine. The `cpp/build` and `python/build` directories within your repository is on a separate mount to avoid conflicting with your local build artifacts. diff --git a/ci/local/build.sh b/ci/local/build.sh deleted file mode 100755 index f6479cd76cc..00000000000 --- a/ci/local/build.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/bash -# Copyright (c) 2019-2022, NVIDIA CORPORATION. - -GIT_DESCRIBE_TAG=`git describe --tags` -MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` - -DOCKER_IMAGE="gpuci/rapidsai:${MINOR_VERSION}-cuda11.5-devel-ubuntu20.04-py3.8" -REPO_PATH=${PWD} -RAPIDS_DIR_IN_CONTAINER="/rapids" -CPP_BUILD_DIR="cpp/build" -PYTHON_BUILD_DIR="python/build" -CONTAINER_SHELL_ONLY=0 - -SHORTHELP="$(basename "$0") [-h] [-H] [-s] [-r ] [-i ]" -LONGHELP="${SHORTHELP} -Build and test your local repository using a base gpuCI Docker image - -where: - -H Show this help text - -r Path to repository (defaults to working directory) - -i Use Docker image (default is ${DOCKER_IMAGE}) - -s Skip building and testing and start an interactive shell in a container of the Docker image -" - -# Limit GPUs available to container based on CUDA_VISIBLE_DEVICES -if [[ -z "${CUDA_VISIBLE_DEVICES}" ]]; then - NVIDIA_VISIBLE_DEVICES="all" -else - NVIDIA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} -fi - -while getopts ":hHr:i:s" option; do - case ${option} in - r) - REPO_PATH=${OPTARG} - ;; - i) - DOCKER_IMAGE=${OPTARG} - ;; - s) - CONTAINER_SHELL_ONLY=1 - ;; - h) - echo "${SHORTHELP}" - exit 0 - ;; - H) - echo "${LONGHELP}" - exit 0 - ;; - *) - echo "ERROR: Invalid flag" - echo "${SHORTHELP}" - exit 1 - ;; - esac -done - -REPO_PATH_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")" -CPP_BUILD_DIR_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")/${CPP_BUILD_DIR}" -PYTHON_BUILD_DIR_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")/${PYTHON_BUILD_DIR}" - - -# BASE_CONTAINER_BUILD_DIR is named after the image name, allowing for -# multiple image builds to coexist on the local filesystem. This will -# be mapped to the typical BUILD_DIR inside of the container. Builds -# running in the container generate build artifacts just as they would -# in a bare-metal environment, and the host filesystem is able to -# maintain the host build in BUILD_DIR as well. -# shellcheck disable=SC2001,SC2005,SC2046 -BASE_CONTAINER_BUILD_DIR=${REPO_PATH}/build_$(echo $(basename "${DOCKER_IMAGE}")|sed -e 's/:/_/g') -CPP_CONTAINER_BUILD_DIR=${BASE_CONTAINER_BUILD_DIR}/cpp -PYTHON_CONTAINER_BUILD_DIR=${BASE_CONTAINER_BUILD_DIR}/python - - -BUILD_SCRIPT="#!/bin/bash -set -e -WORKSPACE=${REPO_PATH_IN_CONTAINER} -PREBUILD_SCRIPT=${REPO_PATH_IN_CONTAINER}/ci/gpu/prebuild.sh -BUILD_SCRIPT=${REPO_PATH_IN_CONTAINER}/ci/gpu/build.sh -if [ -f \${PREBUILD_SCRIPT} ]; then - source \${PREBUILD_SCRIPT} -fi -yes | source \${BUILD_SCRIPT} -" - -if (( CONTAINER_SHELL_ONLY == 0 )); then - COMMAND="${CPP_BUILD_DIR_IN_CONTAINER}/build.sh || bash" -else - COMMAND="bash" -fi - -# Create the build dir for the container to mount, generate the build script inside of it -mkdir -p "${BASE_CONTAINER_BUILD_DIR}" -mkdir -p "${CPP_CONTAINER_BUILD_DIR}" -mkdir -p "${PYTHON_CONTAINER_BUILD_DIR}" -# Create build directories. This is to ensure correct owner for directories. If -# directories don't exist there is side effect from docker volume mounting creating build -# directories owned by root(volume mount point(s)) -mkdir -p "${REPO_PATH}/${CPP_BUILD_DIR}" -mkdir -p "${REPO_PATH}/${PYTHON_BUILD_DIR}" - -echo "${BUILD_SCRIPT}" > "${CPP_CONTAINER_BUILD_DIR}/build.sh" -chmod ugo+x "${CPP_CONTAINER_BUILD_DIR}/build.sh" - -# Mount passwd and group files to docker. This allows docker to resolve username and group -# avoiding these nags: -# * groups: cannot find name for group ID ID -# * I have no name!@id:/$ -# For ldap user user information is not present in system /etc/passwd and /etc/group files. -# Hence we generate dummy files for ldap users which docker uses to resolve username and group - -PASSWD_FILE="/etc/passwd" -GROUP_FILE="/etc/group" - -USER_FOUND=$(grep -wc "$(whoami)" < "$PASSWD_FILE") -if [ "$USER_FOUND" == 0 ]; then - echo "Local User not found, LDAP WAR for docker mounts activated. Creating dummy passwd and group" - echo "files to allow docker resolve username and group" - cp "$PASSWD_FILE" /tmp/passwd - PASSWD_FILE="/tmp/passwd" - cp "$GROUP_FILE" /tmp/group - GROUP_FILE="/tmp/group" - echo "$(whoami):x:$(id -u):$(id -g):$(whoami),,,:$HOME:$SHELL" >> "$PASSWD_FILE" - echo "$(whoami):x:$(id -g):" >> "$GROUP_FILE" -fi - -# Run the generated build script in a container -docker pull "${DOCKER_IMAGE}" - -DOCKER_MAJOR=$(docker -v|sed 's/[^[0-9]*\([0-9]*\).*/\1/') -GPU_OPTS="--gpus device=${NVIDIA_VISIBLE_DEVICES}" -if [ "$DOCKER_MAJOR" -lt 19 ] -then - GPU_OPTS="--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES='${NVIDIA_VISIBLE_DEVICES}'" -fi - -docker run --rm -it ${GPU_OPTS} \ - -u "$(id -u)":"$(id -g)" \ - -v "${REPO_PATH}":"${REPO_PATH_IN_CONTAINER}" \ - -v "${CPP_CONTAINER_BUILD_DIR}":"${CPP_BUILD_DIR_IN_CONTAINER}" \ - -v "${PYTHON_CONTAINER_BUILD_DIR}":"${PYTHON_BUILD_DIR_IN_CONTAINER}" \ - -v "$PASSWD_FILE":/etc/passwd:ro \ - -v "$GROUP_FILE":/etc/group:ro \ - --cap-add=SYS_PTRACE \ - "${DOCKER_IMAGE}" bash -c "${COMMAND}" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 96099b0512d..d2be7d5f222 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -53,7 +53,6 @@ sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' fetch_rapids.cmake # cmake-format rapids-cmake definitions -sed_runner 's/'"branch-.*\/cmake-format-rapids-cmake.json"'/'"branch-${NEXT_SHORT_TAG}\/cmake-format-rapids-cmake.json"'/g' ci/checks/style.sh sed_runner 's/'"branch-.*\/cmake-format-rapids-cmake.json"'/'"branch-${NEXT_SHORT_TAG}\/cmake-format-rapids-cmake.json"'/g' ci/check_style.sh # doxyfile update @@ -81,10 +80,6 @@ sed_runner "s/cudf=${CURRENT_SHORT_TAG}/cudf=${NEXT_SHORT_TAG}/g" README.md sed_runner "s/CUDF_TAG branch-${CURRENT_SHORT_TAG}/CUDF_TAG branch-${NEXT_SHORT_TAG}/" cpp/examples/basic/CMakeLists.txt sed_runner "s/CUDF_TAG branch-${CURRENT_SHORT_TAG}/CUDF_TAG branch-${NEXT_SHORT_TAG}/" cpp/examples/strings/CMakeLists.txt -# ucx-py version update -sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/gpu/build.sh -sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/gpu/java.sh - # Need to distutils-normalize the original version NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))") diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index d6681881419..0be72486319 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -79,7 +79,7 @@ if [[ "${RAPIDS_BUILD_TYPE}" == "nightly" ]]; then ${COMPUTE_SANITIZER_CMD} ${gt} | tee "${RAPIDS_TESTS_DIR}${test_name}.cs.log" done unset GTEST_CUDF_RMM_MODE - # TODO: test-results/*.cs.log are processed in gpuci + # TODO: test-results/*.cs.log are processed in CI fi rapids-logger "Test script exiting with value: $EXITCODE" diff --git a/ci/test_python_cudf.sh b/ci/test_python_cudf.sh index 337ef38cf97..bb33d8473ce 100755 --- a/ci/test_python_cudf.sh +++ b/ci/test_python_cudf.sh @@ -13,7 +13,6 @@ set +e rapids-logger "pytest cudf" pushd python/cudf/cudf -# (TODO: Copied the comment below from gpuCI, need to verify on GitHub Actions) # It is essential to cd into python/cudf/cudf as `pytest-xdist` + `coverage` seem to work only at this directory level. pytest \ --cache-clear \ diff --git a/conda/recipes/dask-cudf/run_test.sh b/conda/recipes/dask-cudf/run_test.sh index 78be90757a2..0c2f628dcf2 100644 --- a/conda/recipes/dask-cudf/run_test.sh +++ b/conda/recipes/dask-cudf/run_test.sh @@ -25,11 +25,11 @@ export DASK_STABLE_VERSION="2023.1.1" # Install the conda-forge or nightly version of dask and distributed if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then - gpuci_logger "gpuci_mamba_retry install -c dask/label/dev 'dask/label/dev::dask' 'dask/label/dev::distributed'" - gpuci_mamba_retry install -c dask/label/dev "dask/label/dev::dask" "dask/label/dev::distributed" + rapids-logger "rapids-mamba-retry install -c dask/label/dev 'dask/label/dev::dask' 'dask/label/dev::distributed'" + rapids-mamba-retry install -c dask/label/dev "dask/label/dev::dask" "dask/label/dev::distributed" else - gpuci_logger "gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall" - gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall + rapids-logger "rapids-mamba-retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall" + rapids-mamba-retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall fi logger "python -c 'import dask_cudf'"