Add compute-sanitizer github workflow action to nightly tests (#12800)

Adds github workflow action to the nightly tests for running `compute-sanitizer` on the libcudf gtests.

Reference: #12530

Authors:
  - David Wendt (https://github.com/davidwendt)
  - AJ Schmidt (https://github.com/ajschmidt8)

Approvers:
  - AJ Schmidt (https://github.com/ajschmidt8)
  - Vyas Ramasubramani (https://github.com/vyasr)
  - Bradley Dice (https://github.com/bdice)

URL: #12800

.github/workflows/test.yaml

@@ -22,6 +22,18 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+  conda-cpp-memcheck-tests:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: nightly
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      node_type: "gpu-latest-1"
+      arch: "amd64"
+      container_image: "rapidsai/ci:latest"
+      run_script: "ci/test_cpp_memcheck.sh"
   conda-python-cudf-tests:
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]

ci/test_cpp.sh

@@ -1,35 +1,7 @@
 #!/bin/bash
 # Copyright (c) 2022-2023, NVIDIA CORPORATION.
 
-set -euo pipefail
-
-. /opt/conda/etc/profile.d/conda.sh
-
-rapids-logger "Generate C++ testing dependencies"
-rapids-dependency-file-generator \
-  --output conda \
-  --file_key test_cpp \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
-
-rapids-mamba-retry env create --force -f env.yaml -n test
-
-# Temporarily allow unbound variables for conda activation.
-set +u
-conda activate test
-set -u
-
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
-mkdir -p "${RAPIDS_TESTS_DIR}"
-
-rapids-print-env
-
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  libcudf libcudf_kafka libcudf-tests
-
-rapids-logger "Check GPU usage"
-nvidia-smi
+source "$(dirname "$0")/test_cpp_common.sh"
 
 EXITCODE=0
 trap "EXITCODE=1" ERR

ci/test_cpp_common.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+. /opt/conda/etc/profile.d/conda.sh
+
+rapids-logger "Generate C++ testing dependencies"
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key test_cpp \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
+
+rapids-mamba-retry env create --force -f env.yaml -n test
+
+# Temporarily allow unbound variables for conda activation.
+set +u
+conda activate test
+set -u
+
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
+mkdir -p "${RAPIDS_TESTS_DIR}"
+
+rapids-print-env
+
+rapids-mamba-retry install \
+  --channel "${CPP_CHANNEL}" \
+  libcudf libcudf_kafka libcudf-tests
+
+rapids-logger "Check GPU usage"
+nvidia-smi

ci/test_cpp_memcheck.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+source "$(dirname "$0")/test_cpp_common.sh"
+
+EXITCODE=0
+trap "EXITCODE=1" ERR
+set +e
+
+# Run gtests with compute-sanitizer
+rapids-logger "Memcheck gtests with rmm_mode=cuda"
+export GTEST_CUDF_RMM_MODE=cuda
+COMPUTE_SANITIZER_CMD="compute-sanitizer --tool memcheck"
+for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do
+    test_name=$(basename ${gt})
+    if [[ "$test_name" == "ERROR_TEST" ]] || [[ "$test_name" == "STREAM_IDENTIFICATION_TEST" ]]; then
+        continue
+    fi
+    echo "Running compute-sanitizer on $test_name"
+    ${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml"
+done
+unset GTEST_CUDF_RMM_MODE
+
+rapids-logger "Test script exiting with value: $EXITCODE"
+exit ${EXITCODE}

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -16,6 +16,7 @@ dependencies:
 - cmake>=3.23.1,!=3.25.0
 - cubinlinker
 - cuda-python>=11.7.1,<12.0
+- cuda-sanitizer-api=11.8.86
 - cudatoolkit=11.8
 - cupy>=9.5.0,<12.0.0a0
 - cxx-compiler

dependencies.yaml

@@ -13,12 +13,14 @@ files:
       - notebooks
       - py_version
       - run
+      - test_cpp
       - test_python
   test_cpp:
     output: none
     includes:
       - cudatoolkit
       - libidentify_stream_usage_build
+      - test_cpp
   test_python:
     output: none
     includes:
@@ -266,6 +268,16 @@ dependencies:
               arch: aarch64
             packages:
               - cupy-cuda11x -f https://pip.cupy.dev/aarch64 # TODO: Verify that this works.
+  test_cpp:
+    specific:
+      - output_types: conda
+        matrices:
+          - matrix:
+              cuda: "11.8"
+            packages:
+              - cuda-sanitizer-api=11.8.86
+          - matrix:
+            packages:
   test_java:
     common:
       - output_types: conda