Merge remote-tracking branch 'upstream/main' into apachegh-41664-pret…

…ty-print-non-cpu
jorisvandenbossche · Jun 12, 2024 · 9ab291d · 9ab291d
2 parents 1c09d52 + 383ffe0
commit 9ab291d
Show file tree

Hide file tree

Showing 1,175 changed files with 43,607 additions and 37,954 deletions.
diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
@@ -197,9 +197,7 @@ jobs:
         mingw-n-bits:
           - 64
         ruby-version:
-          # TODO: Use the latest Ruby again when we fix GH-39130.
-          # - ruby
-          - "3.1"
+          - ruby
     env:
       ARROW_BUILD_STATIC: OFF
       ARROW_BUILD_TESTS: OFF

diff --git a/.gitignore b/.gitignore
@@ -102,8 +102,8 @@ __debug_bin
 .envrc
 
 # Develocity
-.mvn/.gradle-enterprise/
-.mvn/.develocity/
+java/.mvn/.gradle-enterprise/
+java/.mvn/.develocity/
 
 # rat
 filtered_rat.txt

diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat
@@ -66,6 +66,9 @@ set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.txt
 @rem Force conda to use conda-forge
 conda config --add channels conda-forge
 conda config --remove channels defaults
+@rem Ensure using the latest information. If there are invalid caches,
+@rem mamba may use invalid download URL.
+mamba clean --all -y
 @rem Arrow conda environment
 mamba create -n arrow -y -c conda-forge ^
   --file=ci\conda_env_python.txt ^

diff --git a/ci/scripts/java_full_build.sh b/ci/scripts/java_full_build.sh
@@ -49,21 +49,13 @@ fi
 # build the entire project
 mvn clean \
     install \
-    assembly:single \
-    source:jar \
-    javadoc:jar \
     -Papache-release \
     -Parrow-c-data \
     -Parrow-jni \
     -Darrow.cpp.build.dir=$dist_dir \
-    -Darrow.c.jni.dist.dir=$dist_dir \
-    -DdescriptorId=source-release
+    -Darrow.c.jni.dist.dir=$dist_dir
 
 # copy all jar, zip and pom files to the distribution folder
-find . \
-     "(" -name "*-javadoc.jar" -o -name "*-sources.jar" ")" \
-     -exec echo {} ";" \
-     -exec cp {} $dist_dir ";"
 find ~/.m2/repository/org/apache/arrow \
      "(" \
      -name "*.jar" -o \

diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh
@@ -160,6 +160,26 @@ export CMAKE_PREFIX_PATH=/tmp/arrow-dist
 pushd /arrow/python
 python setup.py bdist_wheel
 
+echo "=== Strip symbols from wheel ==="
+mkdir -p dist/temp-fix-wheel
+mv dist/pyarrow-*.whl dist/temp-fix-wheel
+
+pushd dist/temp-fix-wheel
+wheel_name=$(ls pyarrow-*.whl)
+# Unzip and remove old wheel
+unzip $wheel_name
+rm $wheel_name
+for filename in $(ls pyarrow/*.so pyarrow/*.so.*); do
+    echo "Stripping debug symbols from: $filename";
+    strip --strip-debug $filename
+done
+# Zip wheel again after stripping symbols
+zip -r $wheel_name .
+mv $wheel_name ..
+popd
+
+rm -rf dist/temp-fix-wheel
+
 echo "=== (${PYTHON_VERSION}) Tag the wheel with manylinux${MANYLINUX_VERSION} ==="
 auditwheel repair -L . dist/pyarrow-*.whl -w repaired_wheels
 popd
diff --git a/cpp/cmake_modules/Findutf8proc.cmake b/cpp/cmake_modules/Findutf8proc.cmake
@@ -19,7 +19,7 @@ if(utf8proc_FOUND)
   return()
 endif()
 
-if(ARROW_PACKAGE_KIND STREQUAL "vcpkg")
+if(ARROW_PACKAGE_KIND STREQUAL "vcpkg" OR VCPKG_TOOLCHAIN)
   set(find_package_args "")
   if(utf8proc_FIND_VERSION)
     list(APPEND find_package_args ${utf8proc_FIND_VERSION})

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -2819,11 +2819,13 @@ macro(build_utf8proc)
 endmacro()
 
 if(ARROW_WITH_UTF8PROC)
-  resolve_dependency(utf8proc
-                     PC_PACKAGE_NAMES
-                     libutf8proc
-                     REQUIRED_VERSION
-                     "2.2.0")
+  set(utf8proc_resolve_dependency_args utf8proc PC_PACKAGE_NAMES libutf8proc)
+  if(NOT VCPKG_TOOLCHAIN)
+    # utf8proc in vcpkg doesn't provide version information:
+    # https://github.com/microsoft/vcpkg/issues/39176
+    list(APPEND utf8proc_resolve_dependency_args REQUIRED_VERSION "2.2.0")
+  endif()
+  resolve_dependency(${utf8proc_resolve_dependency_args})
 endif()
 
 macro(build_cares)
@@ -4611,8 +4613,11 @@ macro(build_opentelemetry)
   set(_OPENTELEMETRY_LIBS
       common
       http_client_curl
+      logs
+      ostream_log_record_exporter
       ostream_span_exporter
       otlp_http_client
+      otlp_http_log_record_exporter
       otlp_http_exporter
       otlp_recordable
       proto
@@ -4645,6 +4650,14 @@ macro(build_opentelemetry)
       set(_OPENTELEMETRY_STATIC_LIBRARY
           "${OPENTELEMETRY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}opentelemetry_exporter_otlp_http${CMAKE_STATIC_LIBRARY_SUFFIX}"
       )
+    elseif(_OPENTELEMETRY_LIB STREQUAL "otlp_http_log_record_exporter")
+      set(_OPENTELEMETRY_STATIC_LIBRARY
+          "${OPENTELEMETRY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}opentelemetry_exporter_otlp_http_log${CMAKE_STATIC_LIBRARY_SUFFIX}"
+      )
+    elseif(_OPENTELEMETRY_LIB STREQUAL "ostream_log_record_exporter")
+      set(_OPENTELEMETRY_STATIC_LIBRARY
+          "${OPENTELEMETRY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}opentelemetry_exporter_ostream_logs${CMAKE_STATIC_LIBRARY_SUFFIX}"
+      )
     else()
       set(_OPENTELEMETRY_STATIC_LIBRARY
           "${OPENTELEMETRY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}opentelemetry_${_OPENTELEMETRY_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}"
@@ -4679,9 +4692,16 @@ macro(build_opentelemetry)
                       IMPORTED_LOCATION)
   list(APPEND
        OPENTELEMETRY_CMAKE_ARGS
-       -DWITH_OTLP=ON
        -DWITH_OTLP_HTTP=ON
        -DWITH_OTLP_GRPC=OFF
+       # Disabled because it seemed to cause linking errors. May be worth a closer look.
+       -DWITH_FUNC_TESTS=OFF
+       # These options are slated for removal in v1.14 and their features are deemed stable
+       # as of v1.13. However, setting their corresponding ENABLE_* macros in headers seems
+       # finicky - resulting in build failures or ABI-related runtime errors during HTTP
+       # client initialization. There may still be a solution, but we disable them for now.
+       -DWITH_OTLP_HTTP_SSL_PREVIEW=OFF
+       -DWITH_OTLP_HTTP_SSL_TLS_PREVIEW=OFF
        "-DProtobuf_INCLUDE_DIR=${OPENTELEMETRY_PROTOBUF_INCLUDE_DIR}"
        "-DProtobuf_LIBRARY=${OPENTELEMETRY_PROTOBUF_INCLUDE_DIR}"
        "-DProtobuf_PROTOC_EXECUTABLE=${OPENTELEMETRY_PROTOC_EXECUTABLE}")
@@ -4755,19 +4775,25 @@ macro(build_opentelemetry)
   target_link_libraries(opentelemetry-cpp::resources INTERFACE opentelemetry-cpp::common)
   target_link_libraries(opentelemetry-cpp::trace INTERFACE opentelemetry-cpp::common
                                                            opentelemetry-cpp::resources)
+  target_link_libraries(opentelemetry-cpp::logs INTERFACE opentelemetry-cpp::common
+                                                          opentelemetry-cpp::resources)
   target_link_libraries(opentelemetry-cpp::http_client_curl
-                        INTERFACE opentelemetry-cpp::ext CURL::libcurl)
+                        INTERFACE opentelemetry-cpp::common opentelemetry-cpp::ext
+                                  CURL::libcurl)
   target_link_libraries(opentelemetry-cpp::proto INTERFACE ${ARROW_PROTOBUF_LIBPROTOBUF})
   target_link_libraries(opentelemetry-cpp::otlp_recordable
-                        INTERFACE opentelemetry-cpp::trace opentelemetry-cpp::resources
-                                  opentelemetry-cpp::proto)
+                        INTERFACE opentelemetry-cpp::logs opentelemetry-cpp::trace
+                                  opentelemetry-cpp::resources opentelemetry-cpp::proto)
   target_link_libraries(opentelemetry-cpp::otlp_http_client
-                        INTERFACE opentelemetry-cpp::sdk opentelemetry-cpp::proto
+                        INTERFACE opentelemetry-cpp::common opentelemetry-cpp::proto
                                   opentelemetry-cpp::http_client_curl
                                   nlohmann_json::nlohmann_json)
   target_link_libraries(opentelemetry-cpp::otlp_http_exporter
                         INTERFACE opentelemetry-cpp::otlp_recordable
                                   opentelemetry-cpp::otlp_http_client)
+  target_link_libraries(opentelemetry-cpp::otlp_http_log_record_exporter
+                        INTERFACE opentelemetry-cpp::otlp_recordable
+                                  opentelemetry-cpp::otlp_http_client)
 
   foreach(_OPENTELEMETRY_LIB ${_OPENTELEMETRY_LIBS})
     add_dependencies(opentelemetry-cpp::${_OPENTELEMETRY_LIB} opentelemetry_ep)
@@ -4789,7 +4815,11 @@ if(ARROW_WITH_OPENTELEMETRY)
   set(opentelemetry-cpp_SOURCE "AUTO")
   resolve_dependency(opentelemetry-cpp)
   set(ARROW_OPENTELEMETRY_LIBS
-      opentelemetry-cpp::trace opentelemetry-cpp::ostream_span_exporter
+      opentelemetry-cpp::trace
+      opentelemetry-cpp::logs
+      opentelemetry-cpp::otlp_http_log_record_exporter
+      opentelemetry-cpp::ostream_log_record_exporter
+      opentelemetry-cpp::ostream_span_exporter
       opentelemetry-cpp::otlp_http_exporter)
   get_target_property(OPENTELEMETRY_INCLUDE_DIR opentelemetry-cpp::api
                       INTERFACE_INCLUDE_DIRECTORIES)

diff --git a/cpp/cmake_modules/Usevcpkg.cmake b/cpp/cmake_modules/Usevcpkg.cmake
@@ -237,9 +237,6 @@ set(LZ4_ROOT
     CACHE STRING "")
 
 if(CMAKE_HOST_WIN32)
-  set(utf8proc_MSVC_STATIC_LIB_SUFFIX
-      ""
-      CACHE STRING "")
   set(LZ4_MSVC_LIB_PREFIX
       ""
       CACHE STRING "")

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
@@ -200,22 +200,29 @@ function(arrow_add_object_library PREFIX)
   set(SOURCES ${ARGN})
   string(TOLOWER "${PREFIX}" prefix)
   if(WIN32)
-    add_library(${prefix}_shared OBJECT ${SOURCES})
-    add_library(${prefix}_static OBJECT ${SOURCES})
-    set_target_properties(${prefix}_shared PROPERTIES POSITION_INDEPENDENT_CODE ON)
-    set_target_properties(${prefix}_static PROPERTIES POSITION_INDEPENDENT_CODE ON)
-    target_compile_definitions(${prefix}_shared PRIVATE ARROW_EXPORTING)
-    target_compile_definitions(${prefix}_static PRIVATE ARROW_STATIC)
-    target_compile_features(${prefix}_shared PRIVATE cxx_std_17)
-    target_compile_features(${prefix}_static PRIVATE cxx_std_17)
-    set(${PREFIX}_TARGET_SHARED
-        ${prefix}_shared
-        PARENT_SCOPE)
-    set(${PREFIX}_TARGET_STATIC
-        ${prefix}_static
-        PARENT_SCOPE)
+    set(targets)
+    if(ARROW_BUILD_SHARED)
+      add_library(${prefix}_shared OBJECT ${SOURCES})
+      set_target_properties(${prefix}_shared PROPERTIES POSITION_INDEPENDENT_CODE ON)
+      target_compile_definitions(${prefix}_shared PRIVATE ARROW_EXPORTING)
+      target_compile_features(${prefix}_shared PRIVATE cxx_std_17)
+      set(${PREFIX}_TARGET_SHARED
+          ${prefix}_shared
+          PARENT_SCOPE)
+      list(APPEND targets ${prefix}_shared)
+    endif()
+    if(ARROW_BUILD_STATIC)
+      add_library(${prefix}_static OBJECT ${SOURCES})
+      set_target_properties(${prefix}_static PROPERTIES POSITION_INDEPENDENT_CODE ON)
+      target_compile_definitions(${prefix}_static PRIVATE ARROW_STATIC)
+      target_compile_features(${prefix}_static PRIVATE cxx_std_17)
+      set(${PREFIX}_TARGET_STATIC
+          ${prefix}_static
+          PARENT_SCOPE)
+      list(APPEND targets ${prefix}_static)
+    endif()
     set(${PREFIX}_TARGETS
-        ${prefix}_shared ${prefix}_static
+        ${targets}
         PARENT_SCOPE)
   else()
     add_library(${prefix} OBJECT ${SOURCES})
@@ -515,6 +522,7 @@ set(ARROW_UTIL_SRCS
     util/int_util.cc
     util/io_util.cc
     util/list_util.cc
+    util/logger.cc
     util/logging.cc
     util/key_value_metadata.cc
     util/memory.cc
@@ -620,6 +628,17 @@ if(ARROW_WITH_ZSTD)
   endforeach()
 endif()
 
+if(ARROW_WITH_OPENTELEMETRY)
+  arrow_add_object_library(ARROW_TELEMETRY telemetry/logging.cc)
+
+  foreach(ARROW_TELEMETRY_TARGET ${ARROW_TELEMETRY_TARGETS})
+    target_link_libraries(${ARROW_TELEMETRY_TARGET} PRIVATE ${ARROW_OPENTELEMETRY_LIBS})
+  endforeach()
+else()
+  set(ARROW_TELEMETRY_TARGET_SHARED)
+  set(ARROW_TELEMETRY_TARGET_STATIC)
+endif()
+
 set(ARROW_TESTING_SHARED_LINK_LIBS arrow_shared ${ARROW_GTEST_GTEST})
 set(ARROW_TESTING_SHARED_PRIVATE_LINK_LIBS arrow::flatbuffers RapidJSON)
 set(ARROW_TESTING_STATIC_LINK_LIBS arrow::flatbuffers RapidJSON arrow_static
@@ -1009,6 +1028,7 @@ add_arrow_lib(arrow
               ${ARROW_JSON_TARGET_SHARED}
               ${ARROW_MEMORY_POOL_TARGET_SHARED}
               ${ARROW_ORC_TARGET_SHARED}
+              ${ARROW_TELEMETRY_TARGET_SHARED}
               ${ARROW_UTIL_TARGET_SHARED}
               ${ARROW_VENDORED_TARGET_SHARED}
               ${ARROW_SHARED_PRIVATE_LINK_LIBS}
@@ -1024,6 +1044,7 @@ add_arrow_lib(arrow
               ${ARROW_JSON_TARGET_STATIC}
               ${ARROW_MEMORY_POOL_TARGET_STATIC}
               ${ARROW_ORC_TARGET_STATIC}
+              ${ARROW_TELEMETRY_TARGET_STATIC}
               ${ARROW_UTIL_TARGET_STATIC}
               ${ARROW_VENDORED_TARGET_STATIC}
               ${ARROW_SYSTEM_LINK_LIBS}
@@ -1253,6 +1274,10 @@ if(ARROW_SUBSTRAIT)
   add_subdirectory(engine)
 endif()
 
+if(ARROW_WITH_OPENTELEMETRY)
+  add_subdirectory(telemetry)
+endif()
+
 if(ARROW_TENSORFLOW)
   add_subdirectory(adapters/tensorflow)
 endif()
diff --git a/cpp/src/arrow/array/builder_nested.h b/cpp/src/arrow/array/builder_nested.h
@@ -642,6 +642,8 @@ class ARROW_EXPORT MapBuilder : public ArrayBuilder {
 /// \brief Builder class for fixed-length list array value types
 class ARROW_EXPORT FixedSizeListBuilder : public ArrayBuilder {
  public:
+  using TypeClass = FixedSizeListType;
+
   /// Use this constructor to define the built array's type explicitly. If value_builder
   /// has indeterminate type, this builder will also.
   FixedSizeListBuilder(MemoryPool* pool,