[enhancement](thirdparty) upgrade thirdparty libs (#23290)

protobuf 3.15.0 -> 21.11
glog 0.4.0 -> 0.6.0
lz4 1.9.3 -> 1.9.4
curl 7.79.0 -> 8.2.1
brpc 1.4.0 -> 1.6.0
zstd 1.5.2 -> 1.5.5
arrow 7.0.0 -> 13.0.0
abseil 20220623.1 -> 2023012.3
orc 1.7.2 -> 1.9.0
jemalloc for arrow 5.2.1 -> 5.3.0
xsimd 7.0.0 -> 13.0.0
opentelemetry-proto 0.19.0 -> 1.0.0
opentelemetry 1.8.3 -> 1.10.0

new:
c-ares -> 1.19.1
grpc -> 1.54.3

thirdparty/CHANGELOG.md

@@ -2,6 +2,23 @@
 
 This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}`
 
+## v20230411
+- Modified: protobuf 3.15.0 -> 21.11
+- Modified: glog 0.4.0 -> 0.6.0
+- Modified: lz4 1.9.3 -> 1.9.4
+- Modified: curl 7.79.0 -> 8.2.1
+- Modified: brpc 1.4.0 -> 1.6.0
+- Modified: zstd 1.5.2 -> 1.5.5
+- Modified: arrow 7.0.0 -> 13.0.0
+- Modified: abseil 20220623.1 -> 20230125.3
+- Modified: orc 1.7.2 -> 1.9.0
+- Modified: jemalloc for arrow 5.2.1 -> 5.3.0
+- Modified: xsimd 7.0.0 -> 13.0.0
+- Modified: opentelemetry-proto 0.19.0 -> 1.0.0
+- Modified: opentelemetry 1.8.3 -> 1.10.0
+- Added: c-ares -> 1.19.1
+- Added: grpc -> 1.54.3
+
 ## v20230721
 
 - Modified hadoop libhdfs 3.3.4.4 -> 3.3.4.5

thirdparty/build-thirdparty.sh

@@ -416,41 +416,27 @@ build_thrift() {
 build_protobuf() {
     check_if_source_exist "${PROTOBUF_SOURCE}"
     cd "${TP_SOURCE_DIR}/${PROTOBUF_SOURCE}"
-    rm -fr gmock
-
-    # NOTE(amos): -Wl,--undefined=pthread_create force searching for pthread symbols.
-    # See https://stackoverflow.com/a/65348893/1329147 for detailed explanation.
-    mkdir gmock
-    cd gmock
-    tar xf "${TP_SOURCE_DIR}/${GTEST_NAME}"
-
-    mv "${GTEST_SOURCE}" gtest
-
-    cd "${TP_SOURCE_DIR}/${PROTOBUF_SOURCE}"
-
-    ./autogen.sh
 
     if [[ "${KERNEL}" == 'Darwin' ]]; then
         ldflags="-L${TP_LIB_DIR}"
     else
         ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc -Wl,--undefined=pthread_create"
     fi
 
-    CXXFLAGS="-fPIC -O2 -I${TP_INCLUDE_DIR}" \
-        LDFLAGS="${ldflags}" \
-        ./configure --prefix="${TP_INSTALL_DIR}" --disable-shared --enable-static --with-zlib="${TP_INSTALL_DIR}/include"
+    mkdir -p cmake/build
+    cd cmake/build
 
-    # ATTN: If protoc is not built fully statically the linktime libc may newer than runtime.
-    #       This will casue protoc cannot run
-    #       If you really need to dynamically link protoc, please set the environment variable DYN_LINK_PROTOC=1
-
-    if [[ "${DYN_LINK_PROTOC:-0}" == "1" || "${KERNEL}" == 'Darwin' ]]; then
-        echo "link protoc dynamiclly"
-    else
-        cd src
-        sed -i 's/^AM_LDFLAGS\(.*\)$/AM_LDFLAGS\1 -all-static/' Makefile
-        cd -
-    fi
+    CXXFLAGS="-O2 -I${TP_INCLUDE_DIR}" \
+        LDFLAGS="${ldflags}" \
+        "${CMAKE_CMD}" -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" \
+        -Dprotobuf_USE_EXTERNAL_GTEST=ON \
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+        -Dprotobuf_BUILD_SHARED_LIBS=OFF \
+        -Dprotobuf_BUILD_TESTS=OFF \
+        -Dprotobuf_WITH_ZLIB_DEFAULT=ON \
+        -Dprotobuf_ABSL_PROVIDER=package \
+        -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" ../..
 
     make -j "${PARALLEL}"
     make install
@@ -481,16 +467,28 @@ build_glog() {
     check_if_source_exist "${GLOG_SOURCE}"
     cd "${TP_SOURCE_DIR}/${GLOG_SOURCE}"
 
-    # to generate config.guess and config.sub to support aarch64
-    rm -rf config.*
-    autoreconf -i
+    if [[ "${GLOG_SOURCE}" == "glog-0.4.0" ]]; then
+        # to generate config.guess and config.sub to support aarch64
+        rm -rf config.*
+        autoreconf -i
+
+        CPPFLAGS="-I${TP_INCLUDE_DIR} -fpermissive -fPIC" \
+            LDFLAGS="-L${TP_LIB_DIR}" \
+            ./configure --prefix="${TP_INSTALL_DIR}" --enable-frame-pointers --disable-shared --enable-static
 
-    CPPFLAGS="-I${TP_INCLUDE_DIR} -fpermissive -fPIC" \
+        make -j "${PARALLEL}"
+        make install
+    elif [[ "${GLOG_SOURCE}" == "glog-0.6.0" ]]; then
         LDFLAGS="-L${TP_LIB_DIR}" \
-        ./configure --prefix="${TP_INSTALL_DIR}" --enable-frame-pointers --disable-shared --enable-static
+            "${CMAKE_CMD}" -S . -B build -G "Unix Makefiles" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+            -DWITH_UNWIND=OFF \
+            -DBUILD_SHARED_LIBS=OFF
+
+        cmake --build build --target install
+    fi
 
-    make -j "${PARALLEL}"
-    make install
     strip_lib libglog.a
 }
 
@@ -673,7 +671,8 @@ build_re2() {
     check_if_source_exist "${RE2_SOURCE}"
     cd "${TP_SOURCE_DIR}/${RE2_SOURCE}"
 
-    "${CMAKE_CMD}" -DCMAKE_BUILD_TYPE=Release -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}"
+    "${CMAKE_CMD}" -DCMAKE_BUILD_TYPE=Release -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0 -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+        -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}"
     "${BUILD_SYSTEM}" -j "${PARALLEL}" install
     strip_lib libre2.a
 }
@@ -813,8 +812,12 @@ build_brpc() {
 
     # Currently, BRPC can't be built for static libraries only (without .so). Therefore, we should add `-fPIC`
     # to the dependencies which are required by BRPC. Dependencies: zlib, glog, protobuf, leveldb
+    # If BUILD_SHARED_LIBS=OFF, on centos 5.4 will error: `undefined reference to `google::FlagRegisterer`, no error on MacOS.
+    # If glog is compiled before gflags, the above error will not exist, this works in glog 0.4,
+    # but glog 0.6 enforces dependency on gflags.
+    # glog must be enabled, otherwise error: `flag 'v' was defined more than once` (in files 'glog-0.6.0/src/vlog_is_on.cc' and 'brpc-1.6.0/src/butil/logging.cc')
     LDFLAGS="${ldflags}" \
-        "${CMAKE_CMD}" -G "${GENERATOR}" -DBUILD_SHARED_LIBS=1 -DWITH_GLOG=ON -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
+        "${CMAKE_CMD}" -G "${GENERATOR}" -DBUILD_SHARED_LIBS=ON -DWITH_GLOG=ON -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
         -DCMAKE_LIBRARY_PATH="${TP_INSTALL_DIR}/lib64" -DCMAKE_INCLUDE_PATH="${TP_INSTALL_DIR}/include" \
         -DBUILD_BRPC_TOOLS=OFF \
         -DPROTOBUF_PROTOC_EXECUTABLE="${TP_INSTALL_DIR}/bin/protoc" ..
@@ -939,6 +942,56 @@ build_flatbuffers() {
     cp libflatbuffers.a ../../../installed/lib/libflatbuffers.a
 }
 
+# c-ares
+build_cares() {
+    check_if_source_exist "${CARES_SOURCE}"
+    cd "${TP_SOURCE_DIR}/${CARES_SOURCE}"
+
+    mkdir -p build
+    cd build
+    cmake -DCMAKE_BUILD_TYPE=Release \
+        -DCARES_STATIC=ON \
+        -DCARES_SHARED=OFF \
+        -DCARES_STATIC_PIC=ON \
+        -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" ..
+    make
+    make install
+}
+
+# grpc
+build_grpc() {
+    check_if_source_exist "${GRPC_SOURCE}"
+    cd "${TP_SOURCE_DIR}/${GRPC_SOURCE}"
+
+    mkdir -p cmake/build
+    cd cmake/build
+
+    cmake -DgRPC_INSTALL=ON \
+        -DgRPC_BUILD_TESTS=OFF \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
+        -DgRPC_CARES_PROVIDER=package \
+        -Dc-ares_DIR="${TP_INSTALL_DIR}" \
+        -DgRPC_ABSL_PROVIDER=package \
+        -Dabsl_DIR="${TP_INSTALL_DIR}" \
+        -DgRPC_PROTOBUF_PROVIDER=package \
+        -DProtobuf_DIR="${TP_INSTALL_DIR}" \
+        -DgRPC_RE2_PROVIDER=package \
+        -Dre2_DIR:STRING="${TP_INSTALL_DIR}" \
+        -DgRPC_SSL_PROVIDER=package \
+        -DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}" \
+        -DgRPC_ZLIB_PROVIDER=package \
+        -DZLIB_ROOT="${TP_INSTALL_DIR}" \
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+        ../..
+
+    make -j "${PARALLEL}"
+    make install
+
+    # for grpc > v1.55, cmake 2.22 does not support find_dependency, delete this line after cmake version upgrade.
+    # sed -i 's/find_dependency/find_package/g' "${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake
+}
+
 # arrow
 build_arrow() {
     check_if_source_exist "${ARROW_SOURCE}"
@@ -958,6 +1011,8 @@ build_arrow() {
     export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}"
     export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}"
     export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}"
+    export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}"
+    export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}"
 
     if [[ "${KERNEL}" != 'Darwin' ]]; then
         ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc"
@@ -973,22 +1028,38 @@ build_arrow() {
         -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
         -DCMAKE_INSTALL_LIBDIR=lib64 \
         -DARROW_BOOST_USE_SHARED=OFF \
+        -DARROW_WITH_GRPC=ON \
+        -DgRPC_SOURCE=SYSTEM \
+        -DgRPC_ROOT="${TP_INSTALL_DIR}" \
+        -DARROW_WITH_PROTOBUF=ON \
+        -DProtobuf_SOURCE=SYSTEM \
+        -DProtobuf_LIB="${TP_INSTALL_DIR}/lib/libprotoc.a" -DProtobuf_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
+        -DARROW_FLIGHT=ON \
+        -DARROW_FLIGHT_SQL=ON \
         -DBoost_USE_STATIC_RUNTIME=ON \
         -DARROW_GFLAGS_USE_SHARED=OFF \
         -Dgflags_ROOT="${TP_INSTALL_DIR}" \
         -DGLOG_ROOT="${TP_INSTALL_DIR}" \
         -DRE2_ROOT="${TP_INSTALL_DIR}" \
+        -DZLIB_SOURCE=SYSTEM \
         -DZLIB_LIBRARY="${TP_INSTALL_DIR}/lib/libz.a" -DZLIB_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
+        -DRapidJSON_SOURCE=SYSTEM \
         -DRapidJSON_ROOT="${TP_INSTALL_DIR}" \
         -DORC_ROOT="${TP_INSTALL_DIR}" \
+        -Dxsimd_SOURCE=BUNDLED \
         -DBrotli_SOURCE=BUNDLED \
+        -DARROW_LZ4_USE_SHARED=OFF \
         -DLZ4_LIB="${TP_INSTALL_DIR}/lib/liblz4.a" -DLZ4_INCLUDE_DIR="${TP_INSTALL_DIR}/include/lz4" \
         -DLz4_SOURCE=SYSTEM \
+        -DARROW_ZSTD_USE_SHARED=OFF \
         -DZSTD_LIB="${TP_INSTALL_DIR}/lib/libzstd.a" -DZSTD_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
         -Dzstd_SOURCE=SYSTEM \
         -DSnappy_LIB="${TP_INSTALL_DIR}/lib/libsnappy.a" -DSnappy_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \
         -DSnappy_SOURCE=SYSTEM \
         -DBOOST_ROOT="${TP_INSTALL_DIR}" --no-warn-unused-cli \
+        -Djemalloc_SOURCE=BUNDLED \
+        -DARROW_THRIFT_USE_SHARED=OFF \
+        -DThrift_SOURCE=SYSTEM \
         -DThrift_ROOT="${TP_INSTALL_DIR}" ..
 
     "${BUILD_SYSTEM}" -j "${PARALLEL}"
@@ -1014,8 +1085,9 @@ build_abseil() {
         -DABSL_ENABLE_INSTALL=ON \
         -DBUILD_DEPS=ON \
         -DCMAKE_BUILD_TYPE=Release \
-        -DBUILD_SHARED_LIBS=OFF \
-        -DCMAKE_CXX_STANDARD=11
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+        -DABSL_PROPAGATE_CXX_STD=ON \
+        -DBUILD_SHARED_LIBS=OFF
 
     cmake --build "${BUILD_DIR}" -j "${PARALLEL}"
     cmake --install "${BUILD_DIR}" --prefix "${TP_INSTALL_DIR}"
@@ -1527,8 +1599,19 @@ build_opentelemetry() {
     mkdir -p "${BUILD_DIR}"
     cd "${BUILD_DIR}"
 
-    "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" -DBUILD_TESTING=OFF \
-        -DWITH_OTLP=ON -DWITH_OTLP_GRPC=OFF -DWITH_OTLP_HTTP=ON -DWITH_ZIPKIN=ON -DWITH_EXAMPLES=OFF ..
+    CXXFLAGS="-O2 -I${TP_INCLUDE_DIR}" \
+        LDFLAGS="-L${TP_LIB_DIR}" \
+        "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
+        -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" \
+        -DBUILD_TESTING=OFF \
+        -DWITH_OTLP_GRPC=ON \
+        -DBUILD_SHARED_LIBS=OFF \
+        -DWITH_OTLP_HTTP=ON \
+        -DWITH_ABSEIL=ON \
+        -DWITH_FUNC_TESTS=OFF \
+        -DWITH_ZIPKIN=ON \
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+        -DWITH_EXAMPLES=OFF ..
 
     "${BUILD_SYSTEM}" -j "${PARALLEL}"
     "${BUILD_SYSTEM}" install
@@ -1629,10 +1712,11 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
         lzo2
         zstd
         boost # must before thrift
-        protobuf
+        abseil
         gflags
         gtest
         glog
+        protobuf # after gtest
         rapidjson
         snappy
         gperftools
@@ -1649,8 +1733,9 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
         librdkafka
         flatbuffers
         orc
+        cares
+        grpc # after cares, protobuf
         arrow
-        abseil
         s2
         bitshuffle
         croaringbitmap

thirdparty/download-thirdparty.sh

@@ -231,12 +231,21 @@ cd -
 echo "Finished patching ${ABSEIL_SOURCE}"
 
 # glog patch
-cd "${TP_SOURCE_DIR}/${GLOG_SOURCE}"
-if [[ ! -f "${PATCHED_MARK}" ]]; then
-    patch -p1 <"${TP_PATCH_DIR}/glog-0.4.0.patch"
-    touch "${PATCHED_MARK}"
+if [[ "${GLOG_SOURCE}" == "glog-0.4.0" ]]; then
+    cd "${TP_SOURCE_DIR}/${GLOG_SOURCE}"
+    if [[ ! -f "${PATCHED_MARK}" ]]; then
+        patch -p1 <"${TP_PATCH_DIR}/glog-0.4.0.patch"
+        touch "${PATCHED_MARK}"
+    fi
+    cd -
+elif [[ "${GLOG_SOURCE}" == "glog-0.6.0" ]]; then
+    cd "${TP_SOURCE_DIR}/${GLOG_SOURCE}"
+    if [[ ! -f "${PATCHED_MARK}" ]]; then
+        patch -p1 <"${TP_PATCH_DIR}/glog-0.6.0.patch"
+        touch "${PATCHED_MARK}"
+    fi
+    cd -
 fi
-cd -
 echo "Finished patching ${GLOG_SOURCE}"
 
 # gtest patch
@@ -307,26 +316,27 @@ fi
 echo "Finished patching ${ROCKSDB_SOURCE}"
 
 # opentelemetry patch is used to solve the problem that threadlocal depends on GLIBC_2.18
+# fix error: unknown type name 'uint64_t'
 # see: https://github.com/apache/doris/pull/7911
-if [[ "${OPENTELEMETRY_SOURCE}" == "opentelemetry-cpp-1.8.3" ]]; then
+if [[ "${OPENTELEMETRY_SOURCE}" == "opentelemetry-cpp-1.10.0" ]]; then
     rm -rf "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}/third_party/opentelemetry-proto"/*
     cp -r "${TP_SOURCE_DIR}/${OPENTELEMETRY_PROTO_SOURCE}"/* "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}/third_party/opentelemetry-proto"
     mkdir -p "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}/third_party/opentelemetry-proto/.git"
 
     cd "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}"
     if [[ ! -f "${PATCHED_MARK}" ]]; then
-        patch -p1 <"${TP_PATCH_DIR}/opentelemetry-cpp-1.8.3.patch"
+        patch -p1 <"${TP_PATCH_DIR}/opentelemetry-cpp-1.10.0.patch"
         touch "${PATCHED_MARK}"
     fi
     cd -
 fi
 echo "Finished patching ${OPENTELEMETRY_SOURCE}"
 
 # arrow patch is used to get the raw orc reader for filter prune.
-if [[ "${ARROW_SOURCE}" == "apache-arrow-7.0.0" ]]; then
+if [[ "${ARROW_SOURCE}" == "apache-arrow-12.0.1" ]]; then
     cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}"
     if [[ ! -f "${PATCHED_MARK}" ]]; then
-        patch -p1 <"${TP_PATCH_DIR}/apache-arrow-7.0.0.patch"
+        patch -p1 <"${TP_PATCH_DIR}/apache-arrow-12.0.1.patch"
         touch "${PATCHED_MARK}"
     fi
     cd -
@@ -400,15 +410,3 @@ if [[ "${SIMDJSON_SOURCE}" = "simdjson-3.0.1" ]]; then
     cd -
 fi
 echo "Finished patching ${SIMDJSON_SOURCE}"
-
-if [[ "${BRPC_SOURCE}" == 'brpc-1.4.0' ]]; then
-    cd "${TP_SOURCE_DIR}/${BRPC_SOURCE}"
-    if [[ ! -f "${PATCHED_MARK}" ]]; then
-        for patch_file in "${TP_PATCH_DIR}"/brpc-*; do
-            patch -p1 <"${patch_file}"
-        done
-        touch "${PATCHED_MARK}"
-    fi
-    cd -
-fi
-echo "Finished patching ${BRPC_SOURCE}"

thirdparty/patches/absl.patch

@@ -1,11 +1,11 @@
 diff --git a/absl/container/internal/btree.h b/absl/container/internal/btree.h
-index 01f4e749..3ff8e798 100644
+index d734676a..43216318 100644
 --- a/absl/container/internal/btree.h
 +++ b/absl/container/internal/btree.h
-@@ -75,16 +75,6 @@ namespace absl {
+@@ -76,16 +76,6 @@ namespace absl {
  ABSL_NAMESPACE_BEGIN
  namespace container_internal {
- 
+
 -#ifdef ABSL_BTREE_ENABLE_GENERATIONS
 -#error ABSL_BTREE_ENABLE_GENERATIONS cannot be directly set
 -#elif defined(ABSL_HAVE_ADDRESS_SANITIZER) || \
@@ -18,4 +18,3 @@ index 01f4e749..3ff8e798 100644
 -
  template <typename Compare, typename T, typename U>
  using compare_result_t = absl::result_of_t<const Compare(const T &, const U &)>;
-