From 213ad2249d208b22ec49946f04136d8d4455c3f8 Mon Sep 17 00:00:00 2001 From: David Wendt Date: Thu, 16 Feb 2023 19:44:24 -0500 Subject: [PATCH 01/12] Add compute-sanitizer github workflow action to nightly tests --- .github/workflows/test.yaml | 12 ++++ ci/test_cpp_memcheck.sh | 62 +++++++++++++++++++ .../all_cuda-118_arch-x86_64.yaml | 1 + dependencies.yaml | 9 +++ 4 files changed, 84 insertions(+) create mode 100755 ci/test_cpp_memcheck.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ff19d51f8ef..4d9e97a7b28 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,6 +22,18 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + conda-cpp-memcheck-tests: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.04 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + node_type: "gpu-latest-1" + arch: "amd64" + container_image: "rapidsai/ci:latest" + run_script: "ci/test_cpp_memcheck.sh" conda-python-cudf-tests: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.04 diff --git a/ci/test_cpp_memcheck.sh b/ci/test_cpp_memcheck.sh new file mode 100755 index 00000000000..755cf2f232a --- /dev/null +++ b/ci/test_cpp_memcheck.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) 2022-2023, NVIDIA CORPORATION. + +set -euo pipefail + +. /opt/conda/etc/profile.d/conda.sh + +rapids-logger "Generate C++ testing dependencies" +rapids-dependency-file-generator \ + --output conda \ + --file_key test_cpp \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n test + +# Temporarily allow unbound variables for conda activation. +set +u +conda activate test +set -u + +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ +mkdir -p "${RAPIDS_TESTS_DIR}" + +rapids-print-env + +rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + libcudf libcudf_kafka libcudf-tests + +rapids-logger "Check GPU usage" +nvidia-smi + +EXITCODE=0 +trap "EXITCODE=1" ERR +set +e + +# Run gtests with compute-sanitizer +echo "RAPIDS_BUILD_TYPE=${RAPIDS_BUILD_TYPE}" + +if [[ "${RAPIDS_BUILD_TYPE}" == "nightly" ]]; then + rapids-logger "Memcheck gtests with rmm_mode=cuda" + export GTEST_CUDF_RMM_MODE=cuda + COMPUTE_SANITIZER_CMD="compute-sanitizer --tool memcheck" + for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do + test_name=$(basename ${gt}) + if [[ "$test_name" == "ERROR_TEST" ]]; then + continue + fi + if [[ "$test_name" == "STREAM_IDENTIFICATION_TEST" ]]; then + continue + fi + echo "Running gtest $test_name" + echo "${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR}${test_name}.xml" + ${gt} --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml" + done + unset GTEST_CUDF_RMM_MODE + # TODO: test-results/*.cs.log are processed in CI +fi + +rapids-logger "Test script exiting with value: $EXITCODE" +exit ${EXITCODE} diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 675df3891c3..7f33cdae2cd 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -17,6 +17,7 @@ dependencies: - cubinlinker - cuda-python>=11.7.1,<12.0 - cudatoolkit=11.8 +- cuda-sanitizer-api=11.8.86 - cupy>=9.5.0,<12.0.0a0 - cxx-compiler - cython>=0.29,<0.30 diff --git a/dependencies.yaml b/dependencies.yaml index ae8eac4ea30..c00e17f9395 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -19,6 +19,7 @@ files: includes: - cudatoolkit - libidentify_stream_usage_build + - test_cpp test_python: output: none includes: @@ -266,6 +267,14 @@ dependencies: arch: aarch64 packages: - cupy-cuda11x -f https://pip.cupy.dev/aarch64 # TODO: Verify that this works. + test_cpp: + specific: + - output_types: conda + matrices: + - matrix: + cuda: "11.8" + packages: + - cuda-sanitizer-api=11.8.86 test_java: common: - output_types: conda From e68ecac0d60307190c4f6efaeed78932a0ff5da9 Mon Sep 17 00:00:00 2001 From: David Wendt Date: Thu, 16 Feb 2023 19:52:14 -0500 Subject: [PATCH 02/12] add test_cpp dependency --- dependencies.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dependencies.yaml b/dependencies.yaml index c00e17f9395..0cdb5d79e8b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -13,6 +13,7 @@ files: - notebooks - py_version - run + - test_cpp - test_python test_cpp: output: none From 0881019cd3ba75653c22e1f652ea51371418df28 Mon Sep 17 00:00:00 2001 From: David Wendt Date: Thu, 16 Feb 2023 19:54:41 -0500 Subject: [PATCH 03/12] fix order of dependencies --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 7f33cdae2cd..44d6be65574 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -16,8 +16,8 @@ dependencies: - cmake>=3.23.1,!=3.25.0 - cubinlinker - cuda-python>=11.7.1,<12.0 -- cudatoolkit=11.8 - cuda-sanitizer-api=11.8.86 +- cudatoolkit=11.8 - cupy>=9.5.0,<12.0.0a0 - cxx-compiler - cython>=0.29,<0.30 From 984f060d28da7831561765c87ab4158e9dd0a368 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 17 Feb 2023 06:15:40 -0500 Subject: [PATCH 04/12] Update ci/test_cpp_memcheck.sh --- ci/test_cpp_memcheck.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/test_cpp_memcheck.sh b/ci/test_cpp_memcheck.sh index 755cf2f232a..a5eaf6993eb 100755 --- a/ci/test_cpp_memcheck.sh +++ b/ci/test_cpp_memcheck.sh @@ -51,8 +51,7 @@ if [[ "${RAPIDS_BUILD_TYPE}" == "nightly" ]]; then continue fi echo "Running gtest $test_name" - echo "${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR}${test_name}.xml" - ${gt} --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml" + ${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml" done unset GTEST_CUDF_RMM_MODE # TODO: test-results/*.cs.log are processed in CI From 0d2c323c18a25e0575e77fe6d0ce0ae6e30ac528 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 17 Feb 2023 14:37:13 -0500 Subject: [PATCH 05/12] Update dependencies.yaml Co-authored-by: AJ Schmidt --- dependencies.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index 0cdb5d79e8b..de9795b4b39 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -276,6 +276,8 @@ dependencies: cuda: "11.8" packages: - cuda-sanitizer-api=11.8.86 + - matrix: + packages: test_java: common: - output_types: conda From 10ec293e36437c55fd6aec9262ae39ec2e761960 Mon Sep 17 00:00:00 2001 From: David Wendt Date: Tue, 21 Feb 2023 13:55:19 -0500 Subject: [PATCH 06/12] refactor common test script --- ci/test_cpp.sh | 30 +----------------- ci/test_cpp_common.sh | 32 +++++++++++++++++++ ci/test_cpp_memcheck.sh | 70 +++++++++++------------------------------ 3 files changed, 52 insertions(+), 80 deletions(-) create mode 100644 ci/test_cpp_common.sh diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 983a63d4ce9..bd7a82afbea 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -1,35 +1,7 @@ #!/bin/bash # Copyright (c) 2022-2023, NVIDIA CORPORATION. -set -euo pipefail - -. /opt/conda/etc/profile.d/conda.sh - -rapids-logger "Generate C++ testing dependencies" -rapids-dependency-file-generator \ - --output conda \ - --file_key test_cpp \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml - -rapids-mamba-retry env create --force -f env.yaml -n test - -# Temporarily allow unbound variables for conda activation. -set +u -conda activate test -set -u - -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ -mkdir -p "${RAPIDS_TESTS_DIR}" - -rapids-print-env - -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - libcudf libcudf_kafka libcudf-tests - -rapids-logger "Check GPU usage" -nvidia-smi +source "$(dirname "$0")/test_cpp_common.sh" EXITCODE=0 trap "EXITCODE=1" ERR diff --git a/ci/test_cpp_common.sh b/ci/test_cpp_common.sh new file mode 100644 index 00000000000..c7c095dc4df --- /dev/null +++ b/ci/test_cpp_common.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright (c) 2022-2023, NVIDIA CORPORATION. + +set -euo pipefail + +. /opt/conda/etc/profile.d/conda.sh + +rapids-logger "Generate C++ testing dependencies" +rapids-dependency-file-generator \ + --output conda \ + --file_key test_cpp \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n test + +# Temporarily allow unbound variables for conda activation. +set +u +conda activate test +set -u + +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ +mkdir -p "${RAPIDS_TESTS_DIR}" + +rapids-print-env + +rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + libcudf libcudf_kafka libcudf-tests + +rapids-logger "Check GPU usage" +nvidia-smi diff --git a/ci/test_cpp_memcheck.sh b/ci/test_cpp_memcheck.sh index 755cf2f232a..6da492c2692 100755 --- a/ci/test_cpp_memcheck.sh +++ b/ci/test_cpp_memcheck.sh @@ -1,62 +1,30 @@ #!/bin/bash -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2023, NVIDIA CORPORATION. -set -euo pipefail - -. /opt/conda/etc/profile.d/conda.sh - -rapids-logger "Generate C++ testing dependencies" -rapids-dependency-file-generator \ - --output conda \ - --file_key test_cpp \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml - -rapids-mamba-retry env create --force -f env.yaml -n test - -# Temporarily allow unbound variables for conda activation. -set +u -conda activate test -set -u - -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ -mkdir -p "${RAPIDS_TESTS_DIR}" - -rapids-print-env - -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - libcudf libcudf_kafka libcudf-tests - -rapids-logger "Check GPU usage" -nvidia-smi +source "$(dirname "$0")/test_cpp_common.sh" EXITCODE=0 trap "EXITCODE=1" ERR set +e # Run gtests with compute-sanitizer -echo "RAPIDS_BUILD_TYPE=${RAPIDS_BUILD_TYPE}" - -if [[ "${RAPIDS_BUILD_TYPE}" == "nightly" ]]; then - rapids-logger "Memcheck gtests with rmm_mode=cuda" - export GTEST_CUDF_RMM_MODE=cuda - COMPUTE_SANITIZER_CMD="compute-sanitizer --tool memcheck" - for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do - test_name=$(basename ${gt}) - if [[ "$test_name" == "ERROR_TEST" ]]; then - continue - fi - if [[ "$test_name" == "STREAM_IDENTIFICATION_TEST" ]]; then - continue - fi - echo "Running gtest $test_name" - echo "${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR}${test_name}.xml" - ${gt} --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml" - done - unset GTEST_CUDF_RMM_MODE - # TODO: test-results/*.cs.log are processed in CI -fi +rapids-logger "Memcheck gtests with rmm_mode=cuda" +export GTEST_CUDF_RMM_MODE=cuda +COMPUTE_SANITIZER_CMD="compute-sanitizer --tool memcheck" +for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do + test_name=$(basename ${gt}) + if [[ "$test_name" == "ERROR_TEST" ]]; then + continue + fi + if [[ "$test_name" == "STREAM_IDENTIFICATION_TEST" ]]; then + continue + fi + echo "Running gtest $test_name" + echo "${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR}${test_name}.xml" + ${gt} --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml" +done +unset GTEST_CUDF_RMM_MODE +# TODO: test-results/*.cs.log are processed in CI rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} From 3ae9f2ba0227928f418c5ffd5434931bbb400d9a Mon Sep 17 00:00:00 2001 From: David Wendt Date: Tue, 21 Feb 2023 17:18:22 -0500 Subject: [PATCH 07/12] run basline build --- cpp/include/cudf/types.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/include/cudf/types.hpp b/cpp/include/cudf/types.hpp index 8a1e4c9aee7..3735da3f7e0 100644 --- a/cpp/include/cudf/types.hpp +++ b/cpp/include/cudf/types.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022, NVIDIA CORPORATION. + * Copyright (c) 2018-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -324,7 +324,7 @@ constexpr bool operator==(data_type const& lhs, data_type const& rhs) * @return true `lhs` is not equal to `rhs` * @return false `lhs` is equal to `rhs` */ -inline bool operator!=(data_type const& lhs, data_type const& rhs) { return !(lhs == rhs); } +constexpr bool operator!=(data_type const& lhs, data_type const& rhs) { return !(lhs == rhs); } /** * @brief Returns the size in bytes of elements of the specified `data_type` @@ -338,5 +338,7 @@ inline bool operator!=(data_type const& lhs, data_type const& rhs) { return !(lh */ std::size_t size_of(data_type t); +constexpr int fake_var_for_build_baseline = 2304; ///< temporary + /** @} */ } // namespace cudf From 2bead6a93f2bf641e4dae25082ff58648c95e38b Mon Sep 17 00:00:00 2001 From: David Wendt Date: Wed, 22 Feb 2023 06:25:58 -0500 Subject: [PATCH 08/12] back out temp change --- cpp/include/cudf/types.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cpp/include/cudf/types.hpp b/cpp/include/cudf/types.hpp index 3735da3f7e0..8a1e4c9aee7 100644 --- a/cpp/include/cudf/types.hpp +++ b/cpp/include/cudf/types.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -324,7 +324,7 @@ constexpr bool operator==(data_type const& lhs, data_type const& rhs) * @return true `lhs` is not equal to `rhs` * @return false `lhs` is equal to `rhs` */ -constexpr bool operator!=(data_type const& lhs, data_type const& rhs) { return !(lhs == rhs); } +inline bool operator!=(data_type const& lhs, data_type const& rhs) { return !(lhs == rhs); } /** * @brief Returns the size in bytes of elements of the specified `data_type` @@ -338,7 +338,5 @@ constexpr bool operator!=(data_type const& lhs, data_type const& rhs) { return ! */ std::size_t size_of(data_type t); -constexpr int fake_var_for_build_baseline = 2304; ///< temporary - /** @} */ } // namespace cudf From e3e4050dce96d2113bc50f35d08d116f3795cb7c Mon Sep 17 00:00:00 2001 From: David Wendt Date: Wed, 22 Feb 2023 10:55:33 -0500 Subject: [PATCH 09/12] enable compute-sanitizer --- ci/test_cpp_memcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_cpp_memcheck.sh b/ci/test_cpp_memcheck.sh index 6da492c2692..b804b9d967c 100755 --- a/ci/test_cpp_memcheck.sh +++ b/ci/test_cpp_memcheck.sh @@ -21,7 +21,7 @@ for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do fi echo "Running gtest $test_name" echo "${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR}${test_name}.xml" - ${gt} --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml" + "${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR}${test_name}.xml" done unset GTEST_CUDF_RMM_MODE # TODO: test-results/*.cs.log are processed in CI From 8d522c523cfa86f4c7f1e45f257b148846ff86b0 Mon Sep 17 00:00:00 2001 From: David Wendt Date: Wed, 22 Feb 2023 12:03:02 -0500 Subject: [PATCH 10/12] too much double-quoting --- ci/test_cpp_memcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_cpp_memcheck.sh b/ci/test_cpp_memcheck.sh index b804b9d967c..13329058273 100755 --- a/ci/test_cpp_memcheck.sh +++ b/ci/test_cpp_memcheck.sh @@ -21,7 +21,7 @@ for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do fi echo "Running gtest $test_name" echo "${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR}${test_name}.xml" - "${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR}${test_name}.xml" + ${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml" done unset GTEST_CUDF_RMM_MODE # TODO: test-results/*.cs.log are processed in CI From f50795e1a9de2d2a896b97f779c9e772197f1791 Mon Sep 17 00:00:00 2001 From: David Wendt Date: Wed, 22 Feb 2023 12:37:07 -0500 Subject: [PATCH 11/12] remove TODO comment --- ci/test_cpp_memcheck.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/test_cpp_memcheck.sh b/ci/test_cpp_memcheck.sh index 13329058273..f3dc3b5cd76 100755 --- a/ci/test_cpp_memcheck.sh +++ b/ci/test_cpp_memcheck.sh @@ -24,7 +24,6 @@ for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do ${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml" done unset GTEST_CUDF_RMM_MODE -# TODO: test-results/*.cs.log are processed in CI rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} From 6cd59a3b5e155d0a0f319cf5b588e520f32fb23a Mon Sep 17 00:00:00 2001 From: David Wendt Date: Wed, 22 Feb 2023 13:55:48 -0500 Subject: [PATCH 12/12] removed echo of command --- ci/test_cpp_memcheck.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ci/test_cpp_memcheck.sh b/ci/test_cpp_memcheck.sh index f3dc3b5cd76..0cad4fc3a3f 100755 --- a/ci/test_cpp_memcheck.sh +++ b/ci/test_cpp_memcheck.sh @@ -13,14 +13,10 @@ export GTEST_CUDF_RMM_MODE=cuda COMPUTE_SANITIZER_CMD="compute-sanitizer --tool memcheck" for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do test_name=$(basename ${gt}) - if [[ "$test_name" == "ERROR_TEST" ]]; then + if [[ "$test_name" == "ERROR_TEST" ]] || [[ "$test_name" == "STREAM_IDENTIFICATION_TEST" ]]; then continue fi - if [[ "$test_name" == "STREAM_IDENTIFICATION_TEST" ]]; then - continue - fi - echo "Running gtest $test_name" - echo "${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR}${test_name}.xml" + echo "Running compute-sanitizer on $test_name" ${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml" done unset GTEST_CUDF_RMM_MODE