From 0b55c32c27959cb7789030ae80a257ed9121ab70 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 7 Jun 2022 10:41:30 -0400 Subject: [PATCH] Add `conda` compilers (#702) Our `devel` Docker containers need to be switched to using `conda` compilers to resolve a linking error. `raft` is in those containers, but hasn't yet been built with `conda` compilers. This PR addresses that. These changes won't cleanly merge into `branch-22.08` unfortunately due to the changes in #641, but we can address that another time. Authors: - AJ Schmidt (https://github.com/ajschmidt8) - Corey J. Nolet (https://github.com/cjnolet) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Corey J. Nolet (https://github.com/cjnolet) --- build.sh | 26 +++++++++---------- .../libraft_distance/conda_build_config.yaml | 11 ++++++++ conda/recipes/libraft_distance/meta.yaml | 9 ++++--- .../libraft_headers/conda_build_config.yaml | 11 ++++++++ conda/recipes/libraft_headers/meta.yaml | 9 ++++--- .../libraft_nn/conda_build_config.yaml | 11 ++++++++ conda/recipes/libraft_nn/meta.yaml | 9 ++++--- .../recipes/pylibraft/conda_build_config.yaml | 14 ++++++++++ conda/recipes/pylibraft/meta.yaml | 14 ++++++---- conda/recipes/pyraft/conda_build_config.yaml | 15 +++++++++++ conda/recipes/pyraft/meta.yaml | 14 ++++++---- 11 files changed, 111 insertions(+), 32 deletions(-) create mode 100644 conda/recipes/libraft_distance/conda_build_config.yaml create mode 100644 conda/recipes/libraft_headers/conda_build_config.yaml create mode 100644 conda/recipes/libraft_nn/conda_build_config.yaml create mode 100644 conda/recipes/pylibraft/conda_build_config.yaml diff --git a/build.sh b/build.sh index 6e79bec897..bdaa607691 100755 --- a/build.sh +++ b/build.sh @@ -101,12 +101,12 @@ function cmakeArgs { # There are possible weird edge cases that may cause this regex filter to output nothing and fail silently # the true pipe will catch any weird edge cases that may happen and will cause the program to fall back # on the invalid option error - CMAKE_ARGS=$(echo $ARGS | { grep -Eo "\-\-cmake\-args=\".+\"" || true; }) - if [[ -n ${CMAKE_ARGS} ]]; then - # Remove the full CMAKE_ARGS argument from list of args so that it passes validArgs function - ARGS=${ARGS//$CMAKE_ARGS/} + EXTRA_CMAKE_ARGS=$(echo $ARGS | { grep -Eo "\-\-cmake\-args=\".+\"" || true; }) + if [[ -n ${EXTRA_CMAKE_ARGS} ]]; then + # Remove the full EXTRA_CMAKE_ARGS argument from list of args so that it passes validArgs function + ARGS=${ARGS//$EXTRA_CMAKE_ARGS/} # Filter the full argument down to just the extra string that will be added to cmake call - CMAKE_ARGS=$(echo $CMAKE_ARGS | grep -Eo "\".+\"" | sed -e 's/^"//' -e 's/"$//') + EXTRA_CMAKE_ARGS=$(echo $EXTRA_CMAKE_ARGS | grep -Eo "\".+\"" | sed -e 's/^"//' -e 's/"$//') fi fi } @@ -196,9 +196,9 @@ if [[ ${CMAKE_TARGET} == "" ]]; then CMAKE_TARGET="all" fi -# Append `-DFIND_RAFT_CPP=ON` to CMAKE_ARGS unless a user specified the option. -if [[ "${CMAKE_ARGS}" != *"DFIND_RAFT_CPP"* ]]; then - CMAKE_ARGS="${CMAKE_ARGS} -DFIND_RAFT_CPP=ON" +# Append `-DFIND_RAFT_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. +if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_RAFT_CPP"* ]]; then + EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DFIND_RAFT_CPP=ON" fi # If clean given, run it prior to any other steps @@ -250,7 +250,7 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs || hasArg tests || has -DRAFT_COMPILE_DIST_LIBRARY=${COMPILE_DIST_LIBRARY} \ -DRAFT_USE_FAISS_STATIC=${BUILD_STATIC_FAISS} \ -DRAFT_ENABLE_thrust_DEPENDENCY=${ENABLE_thrust_DEPENDENCY} \ - ${CMAKE_ARGS} + ${EXTRA_CMAKE_ARGS} if [[ ${CMAKE_TARGET} != "" ]]; then echo "-- Compiling targets: ${CMAKE_TARGET}, verbose=${VERBOSE_FLAG}" @@ -266,9 +266,9 @@ fi if (( ${NUMARGS} == 0 )) || hasArg pyraft || hasArg docs; then cd ${REPODIR}/python/raft - python setup.py build_ext -j${PARALLEL_LEVEL:-1} --inplace -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBRAFT_BUILD_DIR} ${CMAKE_ARGS} + python setup.py build_ext -j${PARALLEL_LEVEL:-1} --inplace -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBRAFT_BUILD_DIR} ${EXTRA_CMAKE_ARGS} if [[ ${INSTALL_TARGET} != "" ]]; then - python setup.py install --single-version-externally-managed --record=record.txt -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} ${CMAKE_ARGS} + python setup.py install --single-version-externally-managed --record=record.txt -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} ${EXTRA_CMAKE_ARGS} fi fi @@ -276,9 +276,9 @@ fi if (( ${NUMARGS} == 0 )) || hasArg pylibraft; then cd ${REPODIR}/python/pylibraft - python setup.py build_ext -j${PARALLEL_LEVEL:-1} --inplace -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBRAFT_BUILD_DIR} ${CMAKE_ARGS} + python setup.py build_ext -j${PARALLEL_LEVEL:-1} --inplace -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBRAFT_BUILD_DIR} ${EXTRA_CMAKE_ARGS} if [[ ${INSTALL_TARGET} != "" ]]; then - python setup.py install --single-version-externally-managed --record=record.txt -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} ${CMAKE_ARGS} + python setup.py install --single-version-externally-managed --record=record.txt -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} ${EXTRA_CMAKE_ARGS} fi fi diff --git a/conda/recipes/libraft_distance/conda_build_config.yaml b/conda/recipes/libraft_distance/conda_build_config.yaml new file mode 100644 index 0000000000..322fe6faac --- /dev/null +++ b/conda/recipes/libraft_distance/conda_build_config.yaml @@ -0,0 +1,11 @@ +c_compiler_version: + - 9 + +cxx_compiler_version: + - 9 + +cuda_compiler: + - nvcc + +sysroot_version: + - "2.17" diff --git a/conda/recipes/libraft_distance/meta.yaml b/conda/recipes/libraft_distance/meta.yaml index 9b78bd15f3..31ac73a562 100644 --- a/conda/recipes/libraft_distance/meta.yaml +++ b/conda/recipes/libraft_distance/meta.yaml @@ -18,9 +18,6 @@ build: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} script_env: - - CC - - CXX - - CUDAHOSTCXX - PARALLEL_LEVEL - VERSION_SUFFIX - PROJECT_FLASH @@ -33,9 +30,15 @@ build: - SCCACHE_BUCKET=rapids-sccache - SCCACHE_REGION=us-west-2 - SCCACHE_IDLE_TIMEOUT=32768 + ignore_run_exports_from: + - {{ compiler('cuda') }} requirements: build: + - {{ compiler('c') }} + - {{ compiler('cxx') }} + - {{ compiler('cuda') }} {{ cuda_version }} + - sysroot_{{ target_platform }} {{ sysroot_version }} - cmake>=3.20.1,!=3.23.0 host: - libraft-headers {{ version }} diff --git a/conda/recipes/libraft_headers/conda_build_config.yaml b/conda/recipes/libraft_headers/conda_build_config.yaml new file mode 100644 index 0000000000..322fe6faac --- /dev/null +++ b/conda/recipes/libraft_headers/conda_build_config.yaml @@ -0,0 +1,11 @@ +c_compiler_version: + - 9 + +cxx_compiler_version: + - 9 + +cuda_compiler: + - nvcc + +sysroot_version: + - "2.17" diff --git a/conda/recipes/libraft_headers/meta.yaml b/conda/recipes/libraft_headers/meta.yaml index fd95da66ee..8ff538f2a6 100644 --- a/conda/recipes/libraft_headers/meta.yaml +++ b/conda/recipes/libraft_headers/meta.yaml @@ -18,9 +18,6 @@ build: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} script_env: - - CC - - CXX - - CUDAHOSTCXX - PARALLEL_LEVEL - VERSION_SUFFIX - PROJECT_FLASH @@ -33,9 +30,15 @@ build: - SCCACHE_BUCKET=rapids-sccache - SCCACHE_REGION=us-west-2 - SCCACHE_IDLE_TIMEOUT=32768 + ignore_run_exports_from: + - {{ compiler('cuda') }} requirements: build: + - {{ compiler('c') }} + - {{ compiler('cxx') }} + - {{ compiler('cuda') }} {{ cuda_version }} + - sysroot_{{ target_platform }} {{ sysroot_version }} - cmake>=3.20.1,!=3.23.0 host: - nccl>=2.9.9 diff --git a/conda/recipes/libraft_nn/conda_build_config.yaml b/conda/recipes/libraft_nn/conda_build_config.yaml new file mode 100644 index 0000000000..322fe6faac --- /dev/null +++ b/conda/recipes/libraft_nn/conda_build_config.yaml @@ -0,0 +1,11 @@ +c_compiler_version: + - 9 + +cxx_compiler_version: + - 9 + +cuda_compiler: + - nvcc + +sysroot_version: + - "2.17" diff --git a/conda/recipes/libraft_nn/meta.yaml b/conda/recipes/libraft_nn/meta.yaml index fa3392ddc8..1604c15acc 100644 --- a/conda/recipes/libraft_nn/meta.yaml +++ b/conda/recipes/libraft_nn/meta.yaml @@ -18,9 +18,6 @@ build: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} script_env: - - CC - - CXX - - CUDAHOSTCXX - PARALLEL_LEVEL - VERSION_SUFFIX - PROJECT_FLASH @@ -33,9 +30,15 @@ build: - SCCACHE_BUCKET=rapids-sccache - SCCACHE_REGION=us-west-2 - SCCACHE_IDLE_TIMEOUT=32768 + ignore_run_exports_from: + - {{ compiler('cuda') }} requirements: build: + - {{ compiler('c') }} + - {{ compiler('cxx') }} + - {{ compiler('cuda') }} {{ cuda_version }} + - sysroot_{{ target_platform }} {{ sysroot_version }} - cmake>=3.20.1,!=3.23.0 host: - libraft-headers {{ version }} diff --git a/conda/recipes/pylibraft/conda_build_config.yaml b/conda/recipes/pylibraft/conda_build_config.yaml new file mode 100644 index 0000000000..5c2fa69f8e --- /dev/null +++ b/conda/recipes/pylibraft/conda_build_config.yaml @@ -0,0 +1,14 @@ +c_compiler_version: + - 9 + +cxx_compiler_version: + - 9 + +cuda_compiler: + - nvcc + +sysroot_version: + - "2.17" + +cmake_version: + - ">=3.20.1,!=3.23.0" diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 4576e5146f..d19961d39d 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -18,17 +18,21 @@ source: build: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - CC - - CXX - - VERSION_SUFFIX + ignore_run_exports_from: + - {{ compiler('cuda') }} requirements: build: + - cmake {{ cmake_version }} + - {{ compiler('c') }} + - {{ compiler('cxx') }} + - {{ compiler('cuda') }} {{ cuda_version }} + - sysroot_{{ target_platform }} {{ sysroot_version }} + host: - python x.x - setuptools - cython>=0.29,<0.30 - - cmake>=3.20.1,!=3.23.0 + - cmake {{ cmake_version }} - scikit-build>=0.13.1 - rmm {{ minor_version }} - libraft-headers {{ version }} diff --git a/conda/recipes/pyraft/conda_build_config.yaml b/conda/recipes/pyraft/conda_build_config.yaml index c542be2d20..fdc9a58423 100644 --- a/conda/recipes/pyraft/conda_build_config.yaml +++ b/conda/recipes/pyraft/conda_build_config.yaml @@ -1,2 +1,17 @@ +c_compiler_version: + - 9 + +cxx_compiler_version: + - 9 + +cuda_compiler: + - nvcc + +sysroot_version: + - "2.17" + ucx_version: - "1.12.1" + +cmake_version: + - ">=3.20.1,!=3.23.0" diff --git a/conda/recipes/pyraft/meta.yaml b/conda/recipes/pyraft/meta.yaml index 86159622d3..f0c89f411b 100644 --- a/conda/recipes/pyraft/meta.yaml +++ b/conda/recipes/pyraft/meta.yaml @@ -19,17 +19,21 @@ source: build: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - CC - - CXX - - VERSION_SUFFIX + ignore_run_exports_from: + - {{ compiler('cuda') }} requirements: build: + - cmake {{ cmake_version }} + - {{ compiler('c') }} + - {{ compiler('cxx') }} + - {{ compiler('cuda') }} {{ cuda_version }} + - sysroot_{{ target_platform }} {{ sysroot_version }} + host: - python x.x - setuptools - cython>=0.29,<0.30 - - cmake>=3.20.1,!=3.23.0 + - cmake {{ cmake_version }} - scikit-build>=0.13.1 - rmm {{ minor_version }} - libraft-headers {{ version }}