Test static builds in CI and fix nanoarrow configure (#15437)

Resolves #15275 Resolves #15434 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Jake Awe (https://github.com/AyodeAwe) - Robert Maynard (https://github.com/robertmaynard) - Bradley Dice (https://github.com/bdice) URL: #15437
rapidsai · Apr 2, 2024 · 2584fd9 · 2584fd9
1 parent 13a5c7b
commit 2584fd9
Show file tree

Hide file tree

Showing 6 changed files with 239 additions and 4 deletions.
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -20,6 +20,7 @@ jobs:
       - conda-python-cudf-tests
       - conda-python-other-tests
       - conda-java-tests
+      - static-configure
       - conda-notebook-tests
       - docs-build
       - wheel-build-cudf
@@ -88,6 +89,16 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
       run_script: "ci/test_java.sh"
+  static-configure:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      # Use the wheel container so we can skip conda solves and since our
+      # primary static consumers (Spark) are not in conda anyway.
+      container_image: "rapidsai/ci-wheel:latest"
+      run_script: "ci/configure_cpp_static.sh"
   conda-notebook-tests:
     needs: conda-python-build
     secrets: inherit

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -43,6 +43,16 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
       run_script: "ci/test_cpp_memcheck.sh"
+  static-configure:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      # Use the wheel container so we can skip conda solves and since our
+      # primary static consumers (Spark) are not in conda anyway.
+      container_image: "rapidsai/ci-wheel:latest"
+      run_script: "ci/configure_cpp_static.sh"
   conda-python-cudf-tests:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]

diff --git a/ci/configure_cpp_static.sh b/ci/configure_cpp_static.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+rapids-configure-conda-channels
+
+source rapids-date-string
+
+rapids-logger "Configure static cpp build"
+
+ENV_YAML_DIR="$(mktemp -d)"
+REQUIREMENTS_FILE="${ENV_YAML_DIR}/requirements.txt"
+
+rapids-dependency-file-generator \
+  --output requirements \
+  --file_key test_static_build \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${REQUIREMENTS_FILE}"
+
+python -m pip install -r "${REQUIREMENTS_FILE}"
+pyenv rehash
+
+cmake -S cpp -B build_static -GNinja -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF
diff --git a/cpp/cmake/thirdparty/get_nanoarrow.cmake b/cpp/cmake/thirdparty/get_nanoarrow.cmake
@@ -17,6 +17,25 @@ function(find_and_configure_nanoarrow)
   set(oneValueArgs VERSION FORK PINNED_TAG)
   cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
 
+  # Only run if PKG_VERSION is < 0.5.0
+  if(PKG_VERSION VERSION_LESS 0.5.0)
+    set(patch_files_to_run "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/patches/nanoarrow_cmake.diff")
+    set(patch_issues_to_ref
+        "Fix issues with nanoarrow CMake [https://github.com/apache/arrow-nanoarrow/pull/406]"
+    )
+    set(patch_script "${CMAKE_BINARY_DIR}/rapids-cmake/patches/nanoarrow/patch.cmake")
+    set(log_file "${CMAKE_BINARY_DIR}/rapids-cmake/patches/nanoarrow/log")
+    string(TIMESTAMP current_year "%Y" UTC)
+    configure_file(
+      ${rapids-cmake-dir}/cpm/patches/command_template.cmake.in "${patch_script}" @ONLY
+    )
+  else()
+    message(
+      FATAL_ERROR
+        "Nanoarrow version ${PKG_VERSION} already contains the necessary patch. Please remove this patch from cudf."
+    )
+  endif()
+
   rapids_cpm_find(
     nanoarrow ${PKG_VERSION}
     GLOBAL_TARGETS nanoarrow
@@ -26,6 +45,7 @@ function(find_and_configure_nanoarrow)
     # TODO: Commit hashes are not supported with shallow clones. Can switch this if and when we pin
     # to an actual tag.
     GIT_SHALLOW FALSE
+    PATCH_COMMAND ${CMAKE_COMMAND} -P ${patch_script}
     OPTIONS "BUILD_SHARED_LIBS OFF" "NANOARROW_NAMESPACE cudf"
   )
   set_target_properties(nanoarrow PROPERTIES POSITION_INDEPENDENT_CODE ON)

diff --git a/cpp/cmake/thirdparty/patches/nanoarrow_cmake.diff b/cpp/cmake/thirdparty/patches/nanoarrow_cmake.diff
@@ -0,0 +1,161 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 8714c70..1feec13 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -49,7 +49,6 @@ else()
+ endif()
+
+ option(NANOARROW_CODE_COVERAGE "Enable coverage reporting" OFF)
+-add_library(coverage_config INTERFACE)
+
+ # Avoids a warning about timestamps on downloaded files (prefer new policy
+ # if available))
+@@ -111,6 +110,8 @@ if(NANOARROW_BUNDLE)
+   if(NANOARROW_BUILD_TESTS)
+     include_directories(${CMAKE_BINARY_DIR}/amalgamation)
+     add_library(nanoarrow ${NANOARROW_C_TEMP})
++    add_library(nanoarrow::nanoarrow ALIAS nanoarrow)
++
+     target_compile_definitions(nanoarrow PUBLIC "$<$<CONFIG:Debug>:NANOARROW_DEBUG>")
+   endif()
+
+@@ -120,6 +121,7 @@ if(NANOARROW_BUNDLE)
+ else()
+   add_library(nanoarrow src/nanoarrow/array.c src/nanoarrow/schema.c
+                         src/nanoarrow/array_stream.c src/nanoarrow/utils.c)
++  add_library(nanoarrow::nanoarrow ALIAS nanoarrow)
+
+   target_include_directories(nanoarrow
+                              PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+@@ -154,13 +156,50 @@ else()
+     endif()
+   endif()
+
+-  install(TARGETS nanoarrow DESTINATION lib)
++  install(TARGETS nanoarrow
++          DESTINATION lib
++          EXPORT nanoarrow-exports)
+   install(DIRECTORY src/
+           DESTINATION include
+           FILES_MATCHING
+-          PATTERN "*.h")
++          PATTERN "*.h*")
+   install(FILES ${CMAKE_CURRENT_BINARY_DIR}/generated/nanoarrow_config.h
+           DESTINATION include/nanoarrow)
++
++  # Generate package files for the build and install trees.
++  include(CMakePackageConfigHelpers)
++  include(GNUInstallDirs)
++
++  foreach(tree_type BUILD INSTALL)
++    if(tree_type STREQUAL "BUILD")
++      set(install_location ".")
++    else()
++      set(install_location "${CMAKE_INSTALL_LIBDIR}/cmake/nanoarrow")
++    endif()
++
++    set(build_location "${PROJECT_BINARY_DIR}/${install_location}")
++    write_basic_package_version_file(
++      "${build_location}/nanoarrow-config-version.cmake"
++      VERSION ${nanoarrow_VERSION}
++      # After 1.0.0, we can use `SameMajorVersion` here.
++      COMPATIBILITY ExactVersion)
++    configure_package_config_file("${CMAKE_CURRENT_LIST_DIR}/cmake/config.cmake.in"
++                                  "${build_location}/nanoarrow-config.cmake"
++                                  INSTALL_DESTINATION "${install_location}")
++
++    if(tree_type STREQUAL "BUILD")
++      export(EXPORT nanoarrow-exports
++             FILE "${build_location}/nanoarrow-targets.cmake"
++             NAMESPACE nanoarrow::)
++
++    else()
++      install(DIRECTORY "${build_location}/" DESTINATION "${install_location}")
++      install(EXPORT nanoarrow-exports
++              DESTINATION "${install_location}"
++              FILE "nanoarrow-targets.cmake"
++              NAMESPACE nanoarrow::)
++    endif()
++  endforeach()
+ endif()
+
+ # Always build integration test if building tests
+@@ -215,34 +254,18 @@ if(NANOARROW_BUILD_TESTS)
+                  src/nanoarrow/integration/c_data_integration_test.cc)
+
+   if(NANOARROW_CODE_COVERAGE)
+-    target_compile_options(coverage_config INTERFACE -O0 -g --coverage)
+-    target_link_options(coverage_config INTERFACE --coverage)
+-    target_link_libraries(nanoarrow coverage_config)
++    target_compile_options(nanoarrow PUBLIC -O0 -g --coverage)
++    target_link_options(nanoarrow PUBLIC --coverage)
+   endif()
+
+-  target_link_libraries(utils_test
+-                        nanoarrow
+-                        gtest_main
+-                        ${NANOARROW_ARROW_TARGET}
+-                        coverage_config)
+-  target_link_libraries(buffer_test nanoarrow gtest_main coverage_config)
+-  target_link_libraries(array_test
+-                        nanoarrow
+-                        gtest_main
+-                        ${NANOARROW_ARROW_TARGET}
+-                        coverage_config)
+-  target_link_libraries(schema_test
+-                        nanoarrow
+-                        gtest_main
+-                        ${NANOARROW_ARROW_TARGET}
+-                        coverage_config)
+-  target_link_libraries(array_stream_test nanoarrow gtest_main coverage_config)
+-  target_link_libraries(nanoarrow_hpp_test nanoarrow gtest_main coverage_config)
+-  target_link_libraries(nanoarrow_testing_test
+-                        nanoarrow
+-                        gtest_main
+-                        nlohmann_json::nlohmann_json
+-                        coverage_config)
++  target_link_libraries(utils_test nanoarrow gtest_main ${NANOARROW_ARROW_TARGET})
++  target_link_libraries(buffer_test nanoarrow gtest_main)
++  target_link_libraries(array_test nanoarrow gtest_main ${NANOARROW_ARROW_TARGET})
++  target_link_libraries(schema_test nanoarrow gtest_main ${NANOARROW_ARROW_TARGET})
++  target_link_libraries(array_stream_test nanoarrow gtest_main)
++  target_link_libraries(nanoarrow_hpp_test nanoarrow gtest_main)
++  target_link_libraries(nanoarrow_testing_test nanoarrow gtest_main
++                        nlohmann_json::nlohmann_json)
+   target_link_libraries(c_data_integration_test nanoarrow nanoarrow_c_data_integration
+                         gtest_main)
+
+diff --git a/cmake/config.cmake.in b/cmake/config.cmake.in
+new file mode 100644
+index 0000000..021dc31
+--- /dev/null
++++ b/cmake/config.cmake.in
+@@ -0,0 +1,28 @@
++# Licensed to the Apache Software Foundation (ASF) under one
++# or more contributor license agreements.  See the NOTICE file
++# distributed with this work for additional information
++# regarding copyright ownership.  The ASF licenses this file
++# to you under the Apache License, Version 2.0 (the
++# "License"); you may not use this file except in compliance
++# with the License.  You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing,
++# software distributed under the License is distributed on an
++# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++# KIND, either express or implied.  See the License for the
++# specific language governing permissions and limitations
++# under the License.
++
++
++@PACKAGE_INIT@
++
++cmake_minimum_required(VERSION @CMAKE_MINIMUM_REQUIRED_VERSION@)
++
++include("${CMAKE_CURRENT_LIST_DIR}/nanoarrow-targets.cmake" REQUIRED)
++include("${CMAKE_CURRENT_LIST_DIR}/nanoarrow-config-version.cmake" REQUIRED)
++
++set(${CMAKE_FIND_PACKAGE_NAME}_CONFIG "${CMAKE_CURRENT_LIST_FILE}")
++include(FindPackageHandleStandardArgs)
++find_package_handle_standard_args(${CMAKE_FIND_PACKAGE_NAME} CONFIG_MODE)
diff --git a/dependencies.yaml b/dependencies.yaml
@@ -6,6 +6,7 @@ files:
       cuda: ["11.8", "12.2"]
       arch: [x86_64]
     includes:
+      - build_base
       - build_all
       - build_cpp
       - build_wheels
@@ -27,6 +28,10 @@ files:
       - test_python_cudf
       - test_python_dask_cudf
       - depends_on_cupy
+  test_static_build:
+    output: none
+    includes:
+      - build_base
   test_cpp:
     output: none
     includes:
@@ -45,6 +50,7 @@ files:
   test_java:
     output: none
     includes:
+      - build_base
       - build_all
       - cuda
       - cuda_version
@@ -75,6 +81,7 @@ files:
     extras:
       table: build-system
     includes:
+      - build_base
       - build_python_common
       - build_python_cudf
   py_run_cudf:
@@ -144,6 +151,7 @@ files:
     extras:
       table: build-system
     includes:
+      - build_base
       - build_python_common
   py_run_cudf_kafka:
     output: pyproject
@@ -191,12 +199,16 @@ channels:
   - conda-forge
   - nvidia
 dependencies:
-  build_all:
+  build_base:
     common:
-      - output_types: conda
+      - output_types: [conda, requirements, pyproject]
         packages:
           - &cmake_ver cmake>=3.26.4
           - &ninja ninja
+  build_all:
+    common:
+      - output_types: conda
+        packages:
           - c-compiler
           - cxx-compiler
           - dlpack>=0.8,<1.0
@@ -254,9 +266,7 @@ dependencies:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - *cmake_ver
           - cython>=3.0.3
-          - *ninja
           # Hard pin the patch version used during the build. This must be kept
           # in sync with the version pinned in get_arrow.cmake.
           - pyarrow==14.0.2.*