From 2185268bab2f5890a2c3c9a3c315375a971c4416 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Wed, 23 Aug 2023 20:25:49 +0800 Subject: [PATCH] [enhancement](thirdparty) upgrade thirdparty libs (#23290) protobuf 3.15.0 -> 21.11 glog 0.4.0 -> 0.6.0 lz4 1.9.3 -> 1.9.4 curl 7.79.0 -> 8.2.1 brpc 1.4.0 -> 1.6.0 zstd 1.5.2 -> 1.5.5 arrow 7.0.0 -> 13.0.0 abseil 20220623.1 -> 20230125.3 orc 1.7.2 -> 1.9.0 jemalloc for arrow 5.2.1 -> 5.3.0 xsimd 7.0.0 -> 13.0.0 opentelemetry-proto 0.19.0 -> 1.0.0 opentelemetry 1.8.3 -> 1.10.0 new: c-ares -> 1.19.1 grpc -> 1.54.3 --- thirdparty/CHANGELOG.md | 17 + thirdparty/build-thirdparty.sh | 169 ++++++--- thirdparty/download-thirdparty.sh | 40 +- thirdparty/patches/absl.patch | 7 +- ...-7.0.0.patch => apache-arrow-12.0.1.patch} | 52 +-- thirdparty/patches/brpc-1.5.0-clang16.patch | 22 -- thirdparty/patches/brpc-1.5.0-gcc13.patch | 12 - thirdparty/patches/glog-0.6.0.patch | 354 ++++++++++++++++++ ...3.patch => opentelemetry-cpp-1.10.0.patch} | 97 +++-- thirdparty/vars.sh | 126 ++++--- 10 files changed, 664 insertions(+), 232 deletions(-) rename thirdparty/patches/{apache-arrow-7.0.0.patch => apache-arrow-12.0.1.patch} (83%) delete mode 100644 thirdparty/patches/brpc-1.5.0-clang16.patch delete mode 100644 thirdparty/patches/brpc-1.5.0-gcc13.patch create mode 100644 thirdparty/patches/glog-0.6.0.patch rename thirdparty/patches/{opentelemetry-cpp-1.8.3.patch => opentelemetry-cpp-1.10.0.patch} (93%) diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index c90363fcb752d1..be6bfb8fdd9652 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -2,6 +2,23 @@ This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}` +## v20230411 +- Modified: protobuf 3.15.0 -> 21.11 +- Modified: glog 0.4.0 -> 0.6.0 +- Modified: lz4 1.9.3 -> 1.9.4 +- Modified: curl 7.79.0 -> 8.2.1 +- Modified: brpc 1.4.0 -> 1.6.0 +- Modified: zstd 1.5.2 -> 1.5.5 +- Modified: arrow 7.0.0 -> 13.0.0 +- Modified: abseil 20220623.1 -> 20230125.3 +- Modified: orc 1.7.2 -> 1.9.0 +- Modified: jemalloc for arrow 5.2.1 -> 5.3.0 +- Modified: xsimd 7.0.0 -> 13.0.0 +- Modified: opentelemetry-proto 0.19.0 -> 1.0.0 +- Modified: opentelemetry 1.8.3 -> 1.10.0 +- Added: c-ares -> 1.19.1 +- Added: grpc -> 1.54.3 + ## v20230721 - Modified hadoop libhdfs 3.3.4.4 -> 3.3.4.5 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index ec8aa049b96c1d..fa329f48de1205 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -416,19 +416,6 @@ build_thrift() { build_protobuf() { check_if_source_exist "${PROTOBUF_SOURCE}" cd "${TP_SOURCE_DIR}/${PROTOBUF_SOURCE}" - rm -fr gmock - - # NOTE(amos): -Wl,--undefined=pthread_create force searching for pthread symbols. - # See https://stackoverflow.com/a/65348893/1329147 for detailed explanation. - mkdir gmock - cd gmock - tar xf "${TP_SOURCE_DIR}/${GTEST_NAME}" - - mv "${GTEST_SOURCE}" gtest - - cd "${TP_SOURCE_DIR}/${PROTOBUF_SOURCE}" - - ./autogen.sh if [[ "${KERNEL}" == 'Darwin' ]]; then ldflags="-L${TP_LIB_DIR}" @@ -436,21 +423,20 @@ build_protobuf() { ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc -Wl,--undefined=pthread_create" fi - CXXFLAGS="-fPIC -O2 -I${TP_INCLUDE_DIR}" \ - LDFLAGS="${ldflags}" \ - ./configure --prefix="${TP_INSTALL_DIR}" --disable-shared --enable-static --with-zlib="${TP_INSTALL_DIR}/include" + mkdir -p cmake/build + cd cmake/build - # ATTN: If protoc is not built fully statically the linktime libc may newer than runtime. - # This will casue protoc cannot run - # If you really need to dynamically link protoc, please set the environment variable DYN_LINK_PROTOC=1 - - if [[ "${DYN_LINK_PROTOC:-0}" == "1" || "${KERNEL}" == 'Darwin' ]]; then - echo "link protoc dynamiclly" - else - cd src - sed -i 's/^AM_LDFLAGS\(.*\)$/AM_LDFLAGS\1 -all-static/' Makefile - cd - - fi + CXXFLAGS="-O2 -I${TP_INCLUDE_DIR}" \ + LDFLAGS="${ldflags}" \ + "${CMAKE_CMD}" -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" \ + -Dprotobuf_USE_EXTERNAL_GTEST=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -Dprotobuf_BUILD_SHARED_LIBS=OFF \ + -Dprotobuf_BUILD_TESTS=OFF \ + -Dprotobuf_WITH_ZLIB_DEFAULT=ON \ + -Dprotobuf_ABSL_PROVIDER=package \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" ../.. make -j "${PARALLEL}" make install @@ -481,16 +467,28 @@ build_glog() { check_if_source_exist "${GLOG_SOURCE}" cd "${TP_SOURCE_DIR}/${GLOG_SOURCE}" - # to generate config.guess and config.sub to support aarch64 - rm -rf config.* - autoreconf -i + if [[ "${GLOG_SOURCE}" == "glog-0.4.0" ]]; then + # to generate config.guess and config.sub to support aarch64 + rm -rf config.* + autoreconf -i + + CPPFLAGS="-I${TP_INCLUDE_DIR} -fpermissive -fPIC" \ + LDFLAGS="-L${TP_LIB_DIR}" \ + ./configure --prefix="${TP_INSTALL_DIR}" --enable-frame-pointers --disable-shared --enable-static - CPPFLAGS="-I${TP_INCLUDE_DIR} -fpermissive -fPIC" \ + make -j "${PARALLEL}" + make install + elif [[ "${GLOG_SOURCE}" == "glog-0.6.0" ]]; then LDFLAGS="-L${TP_LIB_DIR}" \ - ./configure --prefix="${TP_INSTALL_DIR}" --enable-frame-pointers --disable-shared --enable-static + "${CMAKE_CMD}" -S . -B build -G "Unix Makefiles" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DWITH_UNWIND=OFF \ + -DBUILD_SHARED_LIBS=OFF + + cmake --build build --target install + fi - make -j "${PARALLEL}" - make install strip_lib libglog.a } @@ -673,7 +671,8 @@ build_re2() { check_if_source_exist "${RE2_SOURCE}" cd "${TP_SOURCE_DIR}/${RE2_SOURCE}" - "${CMAKE_CMD}" -DCMAKE_BUILD_TYPE=Release -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" + "${CMAKE_CMD}" -DCMAKE_BUILD_TYPE=Release -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0 -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" "${BUILD_SYSTEM}" -j "${PARALLEL}" install strip_lib libre2.a } @@ -813,8 +812,12 @@ build_brpc() { # Currently, BRPC can't be built for static libraries only (without .so). Therefore, we should add `-fPIC` # to the dependencies which are required by BRPC. Dependencies: zlib, glog, protobuf, leveldb + # If BUILD_SHARED_LIBS=OFF, on centos 5.4 will error: `undefined reference to `google::FlagRegisterer`, no error on MacOS. + # If glog is compiled before gflags, the above error will not exist, this works in glog 0.4, + # but glog 0.6 enforces dependency on gflags. + # glog must be enabled, otherwise error: `flag 'v' was defined more than once` (in files 'glog-0.6.0/src/vlog_is_on.cc' and 'brpc-1.6.0/src/butil/logging.cc') LDFLAGS="${ldflags}" \ - "${CMAKE_CMD}" -G "${GENERATOR}" -DBUILD_SHARED_LIBS=1 -DWITH_GLOG=ON -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + "${CMAKE_CMD}" -G "${GENERATOR}" -DBUILD_SHARED_LIBS=ON -DWITH_GLOG=ON -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ -DCMAKE_LIBRARY_PATH="${TP_INSTALL_DIR}/lib64" -DCMAKE_INCLUDE_PATH="${TP_INSTALL_DIR}/include" \ -DBUILD_BRPC_TOOLS=OFF \ -DPROTOBUF_PROTOC_EXECUTABLE="${TP_INSTALL_DIR}/bin/protoc" .. @@ -939,6 +942,56 @@ build_flatbuffers() { cp libflatbuffers.a ../../../installed/lib/libflatbuffers.a } +# c-ares +build_cares() { + check_if_source_exist "${CARES_SOURCE}" + cd "${TP_SOURCE_DIR}/${CARES_SOURCE}" + + mkdir -p build + cd build + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCARES_STATIC=ON \ + -DCARES_SHARED=OFF \ + -DCARES_STATIC_PIC=ON \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" .. + make + make install +} + +# grpc +build_grpc() { + check_if_source_exist "${GRPC_SOURCE}" + cd "${TP_SOURCE_DIR}/${GRPC_SOURCE}" + + mkdir -p cmake/build + cd cmake/build + + cmake -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DgRPC_CARES_PROVIDER=package \ + -Dc-ares_DIR="${TP_INSTALL_DIR}" \ + -DgRPC_ABSL_PROVIDER=package \ + -Dabsl_DIR="${TP_INSTALL_DIR}" \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DProtobuf_DIR="${TP_INSTALL_DIR}" \ + -DgRPC_RE2_PROVIDER=package \ + -Dre2_DIR:STRING="${TP_INSTALL_DIR}" \ + -DgRPC_SSL_PROVIDER=package \ + -DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}" \ + -DgRPC_ZLIB_PROVIDER=package \ + -DZLIB_ROOT="${TP_INSTALL_DIR}" \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + ../.. + + make -j "${PARALLEL}" + make install + + # for grpc > v1.55, cmake 2.22 does not support find_dependency, delete this line after cmake version upgrade. + # sed -i 's/find_dependency/find_package/g' "${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake +} + # arrow build_arrow() { check_if_source_exist "${ARROW_SOURCE}" @@ -958,6 +1011,8 @@ build_arrow() { export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}" export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}" export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}" + export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}" + export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}" if [[ "${KERNEL}" != 'Darwin' ]]; then ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" @@ -973,22 +1028,38 @@ build_arrow() { -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ -DCMAKE_INSTALL_LIBDIR=lib64 \ -DARROW_BOOST_USE_SHARED=OFF \ + -DARROW_WITH_GRPC=ON \ + -DgRPC_SOURCE=SYSTEM \ + -DgRPC_ROOT="${TP_INSTALL_DIR}" \ + -DARROW_WITH_PROTOBUF=ON \ + -DProtobuf_SOURCE=SYSTEM \ + -DProtobuf_LIB="${TP_INSTALL_DIR}/lib/libprotoc.a" -DProtobuf_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \ + -DARROW_FLIGHT=ON \ + -DARROW_FLIGHT_SQL=ON \ -DBoost_USE_STATIC_RUNTIME=ON \ -DARROW_GFLAGS_USE_SHARED=OFF \ -Dgflags_ROOT="${TP_INSTALL_DIR}" \ -DGLOG_ROOT="${TP_INSTALL_DIR}" \ -DRE2_ROOT="${TP_INSTALL_DIR}" \ + -DZLIB_SOURCE=SYSTEM \ -DZLIB_LIBRARY="${TP_INSTALL_DIR}/lib/libz.a" -DZLIB_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \ + -DRapidJSON_SOURCE=SYSTEM \ -DRapidJSON_ROOT="${TP_INSTALL_DIR}" \ -DORC_ROOT="${TP_INSTALL_DIR}" \ + -Dxsimd_SOURCE=BUNDLED \ -DBrotli_SOURCE=BUNDLED \ + -DARROW_LZ4_USE_SHARED=OFF \ -DLZ4_LIB="${TP_INSTALL_DIR}/lib/liblz4.a" -DLZ4_INCLUDE_DIR="${TP_INSTALL_DIR}/include/lz4" \ -DLz4_SOURCE=SYSTEM \ + -DARROW_ZSTD_USE_SHARED=OFF \ -DZSTD_LIB="${TP_INSTALL_DIR}/lib/libzstd.a" -DZSTD_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \ -Dzstd_SOURCE=SYSTEM \ -DSnappy_LIB="${TP_INSTALL_DIR}/lib/libsnappy.a" -DSnappy_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \ -DSnappy_SOURCE=SYSTEM \ -DBOOST_ROOT="${TP_INSTALL_DIR}" --no-warn-unused-cli \ + -Djemalloc_SOURCE=BUNDLED \ + -DARROW_THRIFT_USE_SHARED=OFF \ + -DThrift_SOURCE=SYSTEM \ -DThrift_ROOT="${TP_INSTALL_DIR}" .. "${BUILD_SYSTEM}" -j "${PARALLEL}" @@ -1014,8 +1085,9 @@ build_abseil() { -DABSL_ENABLE_INSTALL=ON \ -DBUILD_DEPS=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_CXX_STANDARD=11 + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DBUILD_SHARED_LIBS=OFF cmake --build "${BUILD_DIR}" -j "${PARALLEL}" cmake --install "${BUILD_DIR}" --prefix "${TP_INSTALL_DIR}" @@ -1527,8 +1599,19 @@ build_opentelemetry() { mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" -DBUILD_TESTING=OFF \ - -DWITH_OTLP=ON -DWITH_OTLP_GRPC=OFF -DWITH_OTLP_HTTP=ON -DWITH_ZIPKIN=ON -DWITH_EXAMPLES=OFF .. + CXXFLAGS="-O2 -I${TP_INCLUDE_DIR}" \ + LDFLAGS="-L${TP_LIB_DIR}" \ + "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" \ + -DBUILD_TESTING=OFF \ + -DWITH_OTLP_GRPC=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DWITH_OTLP_HTTP=ON \ + -DWITH_ABSEIL=ON \ + -DWITH_FUNC_TESTS=OFF \ + -DWITH_ZIPKIN=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DWITH_EXAMPLES=OFF .. "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install @@ -1629,10 +1712,11 @@ if [[ "${#packages[@]}" -eq 0 ]]; then lzo2 zstd boost # must before thrift - protobuf + abseil gflags gtest glog + protobuf # after gtest rapidjson snappy gperftools @@ -1649,8 +1733,9 @@ if [[ "${#packages[@]}" -eq 0 ]]; then librdkafka flatbuffers orc + cares + grpc # after cares, protobuf arrow - abseil s2 bitshuffle croaringbitmap diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 6a7e62d0b2f16a..39b97b2c0b62d4 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -231,12 +231,21 @@ cd - echo "Finished patching ${ABSEIL_SOURCE}" # glog patch -cd "${TP_SOURCE_DIR}/${GLOG_SOURCE}" -if [[ ! -f "${PATCHED_MARK}" ]]; then - patch -p1 <"${TP_PATCH_DIR}/glog-0.4.0.patch" - touch "${PATCHED_MARK}" +if [[ "${GLOG_SOURCE}" == "glog-0.4.0" ]]; then + cd "${TP_SOURCE_DIR}/${GLOG_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/glog-0.4.0.patch" + touch "${PATCHED_MARK}" + fi + cd - +elif [[ "${GLOG_SOURCE}" == "glog-0.6.0" ]]; then + cd "${TP_SOURCE_DIR}/${GLOG_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/glog-0.6.0.patch" + touch "${PATCHED_MARK}" + fi + cd - fi -cd - echo "Finished patching ${GLOG_SOURCE}" # gtest patch @@ -307,15 +316,16 @@ fi echo "Finished patching ${ROCKSDB_SOURCE}" # opentelemetry patch is used to solve the problem that threadlocal depends on GLIBC_2.18 +# fix error: unknown type name 'uint64_t' # see: https://github.com/apache/doris/pull/7911 -if [[ "${OPENTELEMETRY_SOURCE}" == "opentelemetry-cpp-1.8.3" ]]; then +if [[ "${OPENTELEMETRY_SOURCE}" == "opentelemetry-cpp-1.10.0" ]]; then rm -rf "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}/third_party/opentelemetry-proto"/* cp -r "${TP_SOURCE_DIR}/${OPENTELEMETRY_PROTO_SOURCE}"/* "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}/third_party/opentelemetry-proto" mkdir -p "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}/third_party/opentelemetry-proto/.git" cd "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}" if [[ ! -f "${PATCHED_MARK}" ]]; then - patch -p1 <"${TP_PATCH_DIR}/opentelemetry-cpp-1.8.3.patch" + patch -p1 <"${TP_PATCH_DIR}/opentelemetry-cpp-1.10.0.patch" touch "${PATCHED_MARK}" fi cd - @@ -323,10 +333,10 @@ fi echo "Finished patching ${OPENTELEMETRY_SOURCE}" # arrow patch is used to get the raw orc reader for filter prune. -if [[ "${ARROW_SOURCE}" == "apache-arrow-7.0.0" ]]; then +if [[ "${ARROW_SOURCE}" == "apache-arrow-12.0.1" ]]; then cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}" if [[ ! -f "${PATCHED_MARK}" ]]; then - patch -p1 <"${TP_PATCH_DIR}/apache-arrow-7.0.0.patch" + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-12.0.1.patch" touch "${PATCHED_MARK}" fi cd - @@ -400,15 +410,3 @@ if [[ "${SIMDJSON_SOURCE}" = "simdjson-3.0.1" ]]; then cd - fi echo "Finished patching ${SIMDJSON_SOURCE}" - -if [[ "${BRPC_SOURCE}" == 'brpc-1.4.0' ]]; then - cd "${TP_SOURCE_DIR}/${BRPC_SOURCE}" - if [[ ! -f "${PATCHED_MARK}" ]]; then - for patch_file in "${TP_PATCH_DIR}"/brpc-*; do - patch -p1 <"${patch_file}" - done - touch "${PATCHED_MARK}" - fi - cd - -fi -echo "Finished patching ${BRPC_SOURCE}" diff --git a/thirdparty/patches/absl.patch b/thirdparty/patches/absl.patch index a5568ea6f7183c..4a16368a9b24d5 100644 --- a/thirdparty/patches/absl.patch +++ b/thirdparty/patches/absl.patch @@ -1,11 +1,11 @@ diff --git a/absl/container/internal/btree.h b/absl/container/internal/btree.h -index 01f4e749..3ff8e798 100644 +index d734676a..43216318 100644 --- a/absl/container/internal/btree.h +++ b/absl/container/internal/btree.h -@@ -75,16 +75,6 @@ namespace absl { +@@ -76,16 +76,6 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace container_internal { - + -#ifdef ABSL_BTREE_ENABLE_GENERATIONS -#error ABSL_BTREE_ENABLE_GENERATIONS cannot be directly set -#elif defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ @@ -18,4 +18,3 @@ index 01f4e749..3ff8e798 100644 - template using compare_result_t = absl::result_of_t; - diff --git a/thirdparty/patches/apache-arrow-7.0.0.patch b/thirdparty/patches/apache-arrow-12.0.1.patch similarity index 83% rename from thirdparty/patches/apache-arrow-7.0.0.patch rename to thirdparty/patches/apache-arrow-12.0.1.patch index 072ea9b567a8c4..9a4f12e14be630 100644 --- a/thirdparty/patches/apache-arrow-7.0.0.patch +++ b/thirdparty/patches/apache-arrow-12.0.1.patch @@ -1,30 +1,30 @@ diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc -index 03243e751..1eb9b2c81 100644 +index ac2a3db64..630d45798 100644 --- a/cpp/src/arrow/adapters/orc/adapter.cc +++ b/cpp/src/arrow/adapters/orc/adapter.cc @@ -47,9 +47,6 @@ #include "arrow/util/visibility.h" #include "orc/Exceptions.hh" - + -// alias to not interfere with nested orc namespace -namespace liborc = orc; - #define ORC_THROW_NOT_OK(s) \ do { \ Status _s = (s); \ -@@ -198,6 +195,8 @@ class ORCFileReader::Impl { +@@ -202,6 +199,8 @@ class ORCFileReader::Impl { return Init(); } - + + virtual liborc::Reader* GetRawORCReader() { return reader_.get(); } + Status Init() { int64_t nstripes = reader_->getNumberOfStripes(); stripes_.resize(nstripes); -@@ -504,6 +503,32 @@ class ORCFileReader::Impl { +@@ -479,6 +478,32 @@ class ORCFileReader::Impl { return Status::OK(); } - + + Result> NextStripeReader( + int64_t batch_size, const std::vector& include_names) { + if (current_row_ >= NumberOfRows()) { @@ -51,32 +51,32 @@ index 03243e751..1eb9b2c81 100644 + pool_); + } + - Status NextStripeReader(int64_t batch_size, std::shared_ptr* out) { - return NextStripeReader(batch_size, {}, out); - } -@@ -531,6 +556,8 @@ Result> ORCFileReader::Open( + Result> NextStripeReader( + int64_t batch_size, const std::vector& include_indices) { + if (current_row_ >= NumberOfRows()) { +@@ -544,6 +569,8 @@ Result> ORCFileReader::Open( return std::move(result); } - + +liborc::Reader* ORCFileReader::GetRawORCReader() { return impl_->GetRawORCReader(); } + Result> ORCFileReader::ReadMetadata() { return impl_->ReadMetadata(); } -@@ -653,6 +680,11 @@ Result> ORCFileReader::NextStripeReader( - return reader; +@@ -605,6 +632,11 @@ Result> ORCFileReader::NextStripeReader( + return impl_->NextStripeReader(batch_size, include_indices); } - + +Result> ORCFileReader::NextStripeReader( + int64_t batch_size, const std::vector& include_names) { + return impl_->NextStripeReader(batch_size, include_names); +} + int64_t ORCFileReader::NumberOfStripes() { return impl_->NumberOfStripes(); } - + int64_t ORCFileReader::NumberOfRows() { return impl_->NumberOfRows(); } diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h -index 223efa515..04e6b0612 100644 +index 013be7860..7fd06bcb8 100644 --- a/cpp/src/arrow/adapters/orc/adapter.h +++ b/cpp/src/arrow/adapters/orc/adapter.h @@ -30,6 +30,10 @@ @@ -87,23 +87,23 @@ index 223efa515..04e6b0612 100644 + +// alias to not interfere with nested orc namespace +namespace liborc = orc; - + namespace arrow { namespace adapters { -@@ -51,6 +55,9 @@ class ARROW_EXPORT ORCFileReader { - static Status Open(const std::shared_ptr& file, MemoryPool* pool, - std::unique_ptr* reader); - +@@ -53,6 +57,9 @@ class ARROW_EXPORT ORCFileReader { + public: + ~ORCFileReader(); + + /// \brief Get ORC reader from inside. + liborc::Reader* GetRawORCReader(); + /// \brief Creates a new ORC reader /// /// \param[in] file the data source -@@ -240,6 +247,19 @@ class ARROW_EXPORT ORCFileReader { - Result> NextStripeReader( - int64_t batch_size, const std::vector& include_indices); - +@@ -174,6 +181,19 @@ class ARROW_EXPORT ORCFileReader { + Result> GetRecordBatchReader( + int64_t batch_size, const std::vector& include_names); + + /// \brief Get a stripe level record batch iterator with specified row count + /// in each record batch. NextStripeReader serves as a fine grain + /// alternative to ReadStripe which may cause OOM issue by loading @@ -119,4 +119,4 @@ index 223efa515..04e6b0612 100644 + /// \brief The number of stripes in the file int64_t NumberOfStripes(); - + diff --git a/thirdparty/patches/brpc-1.5.0-clang16.patch b/thirdparty/patches/brpc-1.5.0-clang16.patch deleted file mode 100644 index 42bb5bc5c9a9db..00000000000000 --- a/thirdparty/patches/brpc-1.5.0-clang16.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/src/bthread/task_group.cpp b/src/bthread/task_group.cpp -index 6f5a4abd..e3aef91c 100644 ---- a/src/bthread/task_group.cpp -+++ b/src/bthread/task_group.cpp -@@ -248,7 +248,7 @@ int TaskGroup::init(size_t runqueue_capacity) { - return 0; - } - --#if defined(__linux__) && defined(__aarch64__) && defined(__clang__) -+#if defined(__clang__) - __attribute__((optnone)) - #endif - void TaskGroup::task_runner(intptr_t skip_remained) { -@@ -570,7 +570,7 @@ void TaskGroup::sched(TaskGroup** pg) { - sched_to(pg, next_tid); - } - --#if defined(__linux__) && defined(__aarch64__) && defined(__clang__) -+#if defined(__clang__) - __attribute__((optnone)) - #endif - void TaskGroup::sched_to(TaskGroup** pg, TaskMeta* next_meta) { diff --git a/thirdparty/patches/brpc-1.5.0-gcc13.patch b/thirdparty/patches/brpc-1.5.0-gcc13.patch deleted file mode 100644 index 0cf7988d1d2cf8..00000000000000 --- a/thirdparty/patches/brpc-1.5.0-gcc13.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/src/brpc/http2.h b/src/brpc/http2.h -index 9a40d40..69d3087 100644 ---- a/src/brpc/http2.h -+++ b/src/brpc/http2.h -@@ -18,6 +18,7 @@ - #ifndef BAIDU_RPC_HTTP2_H - #define BAIDU_RPC_HTTP2_H - -+#include - #include "brpc/http_status_code.h" - - // To baidu-rpc developers: This is a header included by user, don't depend diff --git a/thirdparty/patches/glog-0.6.0.patch b/thirdparty/patches/glog-0.6.0.patch new file mode 100644 index 00000000000000..1997f860cb2939 --- /dev/null +++ b/thirdparty/patches/glog-0.6.0.patch @@ -0,0 +1,354 @@ +diff --git a/src/glog/logging.h.in b/src/glog/logging.h.in +index 95a573b..57d2e9b 100644 +--- a/src/glog/logging.h.in ++++ b/src/glog/logging.h.in +@@ -425,6 +425,12 @@ typedef void(*CustomPrefixCallback)(std::ostream& s, const LogMessageInfo& l, vo + using fLS::FLAGS_##name + #endif + ++// Set max log file num ++DECLARE_int32(log_filenum_quota); ++ ++// Set log file split method ++DECLARE_string(log_split_method); ++ + // Set whether appending a timestamp to the log file name + DECLARE_bool(timestamp_in_logfile_name); + +diff --git a/src/logging.cc b/src/logging.cc +index 4028ccc..fd60fc3 100644 +--- a/src/logging.cc ++++ b/src/logging.cc +@@ -34,6 +34,7 @@ + #include + #include + #include ++#include + #include + #ifdef HAVE_UNISTD_H + # include // For _exit. +@@ -50,6 +51,7 @@ + #include + #include + #include ++#include + #ifdef HAVE_PWD_H + # include + #endif +@@ -204,6 +206,12 @@ GLOG_DEFINE_string(log_backtrace_at, "", + GLOG_DEFINE_bool(log_utc_time, false, + "Use UTC time for logging."); + ++GLOG_DEFINE_string(log_split_method, "day", ++ "split log by size, day, hour"); ++ ++GLOG_DEFINE_int32(log_filenum_quota, 10, ++ "max log file num in log dir"); ++ + // TODO(hamaji): consider windows + #define PATH_SEPARATOR '/' + +@@ -429,6 +437,15 @@ namespace { + + namespace { + ++typedef struct filetime { ++ std::string name; ++ time_t time; ++ ++ bool operator < (const struct filetime& o) const { ++ return o.time > time; ++ } ++} Filetime; ++ + // Encapsulates all file-system related state + class LogFileObject : public base::Logger { + public: +@@ -459,6 +476,7 @@ class LogFileObject : public base::Logger { + // can avoid grabbing a lock. Usually Flush() calls it after + // acquiring lock_. + void FlushUnlocked(); ++ void CheckFileNumQuota(); + + private: + static const uint32 kRolloverAttemptFrequency = 0x20; +@@ -476,6 +494,9 @@ class LogFileObject : public base::Logger { + unsigned int rollover_attempt_; + int64 next_flush_time_; // cycle count at which to flush log + WallTime start_time_; ++ std::list file_list_; ++ bool inited_; ++ struct ::tm tm_time_; + + // Actually create a logfile using the value of base_filename_ and the + // optional argument time_pid_string +@@ -665,7 +686,7 @@ inline void LogDestination::FlushLogFiles(int min_severity) { + // all this stuff. + MutexLock l(&log_mutex); + for (int i = min_severity; i < NUM_SEVERITIES; i++) { +- LogDestination* log = log_destination(i); ++ LogDestination* log = log_destinations_[i]; + if (log != NULL) { + log->logger_->Flush(); + } +@@ -876,9 +897,12 @@ inline void LogDestination::LogToAllLogfiles(LogSeverity severity, + } else if (FLAGS_logtostderr) { // global flag: never log to file + ColoredWriteToStderr(severity, message, len); + } else { +- for (int i = severity; i >= 0; --i) { +- LogDestination::MaybeLogToLogfile(i, timestamp, message, len); +- } ++ if (severity >= 1) { ++ LogDestination::MaybeLogToLogfile(1, timestamp, message, len); ++ LogDestination::MaybeLogToLogfile(0, timestamp, message, len); ++ } else if (severity == 0) { ++ LogDestination::MaybeLogToLogfile(0, timestamp, message, len); ++ } else {} + } + } + +@@ -976,6 +1000,7 @@ LogFileObject::LogFileObject(LogSeverity severity, + file_length_(0), + rollover_attempt_(kRolloverAttemptFrequency-1), + next_flush_time_(0), ++ inited_(false), + start_time_(WallTime_Now()) { + assert(severity >= 0); + assert(severity < NUM_SEVERITIES); +@@ -1045,11 +1070,11 @@ bool LogFileObject::CreateLogfile(const string& time_pid_string) { + string_filename += filename_extension_; + const char* filename = string_filename.c_str(); + //only write to files, create if non-existant. +- int flags = O_WRONLY | O_CREAT; +- if (FLAGS_timestamp_in_logfile_name) { +- //demand that the file is unique for our timestamp (fail if it exists). +- flags = flags | O_EXCL; +- } ++ int flags = O_WRONLY | O_CREAT | O_APPEND; ++ // if (FLAGS_timestamp_in_logfile_name) { ++ // //demand that the file is unique for our timestamp (fail if it exists). ++ // flags = flags | O_EXCL; ++ // } + int fd = open(filename, flags, FLAGS_logfile_mode); + if (fd == -1) return false; + #ifdef HAVE_FCNTL +@@ -1097,6 +1122,10 @@ bool LogFileObject::CreateLogfile(const string& time_pid_string) { + } + } + #endif ++ Filetime ft; ++ ft.name = string_filename; ++ file_list_.push_back(ft); ++ + // We try to create a symlink called ., + // which is easier to use. (Every time we create a new logfile, + // we destroy the old symlink and create a new one, so it always +@@ -1138,6 +1167,59 @@ bool LogFileObject::CreateLogfile(const string& time_pid_string) { + return true; // Everything worked + } + ++void LogFileObject::CheckFileNumQuota() { ++ struct dirent *entry; ++ DIR *dp; ++ ++ const vector & log_dirs = GetLoggingDirectories(); ++ if (log_dirs.size() < 1) return; ++ ++ //fprintf(stderr, "log dir: %s\n", log_dirs[0].c_str()); ++ ++ // list file in log dir ++ dp = opendir(log_dirs[0].c_str()); ++ if (dp == NULL) { ++ fprintf(stderr, "open log dir %s fail\n", log_dirs[0].c_str()); ++ return; ++ } ++ ++ file_list_.clear(); ++ while ((entry = readdir(dp)) != NULL) { ++ if (DT_DIR == entry->d_type || ++ DT_LNK == entry->d_type) { ++ continue; ++ } ++ std::string filename = std::string(entry->d_name); ++ //fprintf(stderr, "filename: %s\n", filename.c_str()); ++ ++ if (filename.find(symlink_basename_ + '.' + LogSeverityNames[severity_]) == 0) { ++ std::string filepath = log_dirs[0] + "/" + filename; ++ ++ struct stat fstat; ++ if (::stat(filepath.c_str(), &fstat) < 0) { ++ fprintf(stderr, "state %s fail\n", filepath.c_str()); ++ closedir(dp); ++ return; ++ } ++ //fprintf(stderr, "filepath: %s\n", filepath.c_str()); ++ ++ Filetime file_time; ++ file_time.time = fstat.st_mtime; ++ file_time.name = filepath; ++ file_list_.push_back(file_time); ++ } ++ } ++ closedir(dp); ++ ++ file_list_.sort(); ++ ++ while (FLAGS_log_filenum_quota > 0 && file_list_.size() >= FLAGS_log_filenum_quota) { ++ // fprintf(stderr, "delete %s\n", file_list_.front().name.c_str()); ++ unlink(file_list_.front().name.c_str()); ++ file_list_.pop_front(); ++ } ++} ++ + void LogFileObject::Write(bool force_flush, + time_t timestamp, + const char* message, +@@ -1149,13 +1231,55 @@ void LogFileObject::Write(bool force_flush, + return; + } + +- if (file_length_ >> 20U >= MaxLogSize() || PidHasChanged()) { ++ struct ::tm tm_time; ++ ++ bool is_split = false; ++ if ("day" == FLAGS_log_split_method) { ++ localtime_r(×tamp, &tm_time); ++ if (tm_time.tm_year != tm_time_.tm_year ++ || tm_time.tm_mon != tm_time_.tm_mon ++ || tm_time.tm_mday != tm_time_.tm_mday) { ++ is_split = true; ++ } ++ } else if ("hour" == FLAGS_log_split_method) { ++ localtime_r(×tamp, &tm_time); ++ if (tm_time.tm_year != tm_time_.tm_year ++ || tm_time.tm_mon != tm_time_.tm_mon ++ || tm_time.tm_mday != tm_time_.tm_mday ++ || tm_time.tm_hour != tm_time_.tm_hour) { ++ is_split = true; ++ } ++ } else if (file_length_ >> 20U >= MaxLogSize()) { ++ // PidHasChanged()) { ++ is_split = true; ++ } ++ ++ if (is_split) { + if (file_ != NULL) fclose(file_); + file_ = NULL; + file_length_ = bytes_since_flush_ = dropped_mem_length_ = 0; + rollover_attempt_ = kRolloverAttemptFrequency - 1; + } + ++ if ((file_ == NULL) && (!inited_) && (FLAGS_log_split_method == "size")) { ++ CheckFileNumQuota(); ++ const char* filename = file_list_.back().name.c_str(); ++ int fd = open(filename, O_WRONLY | O_CREAT | O_APPEND, FLAGS_logfile_mode); ++ if (fd != -1) { ++#ifdef HAVE_FCNTL ++ // Mark the file close-on-exec. We don't really care if this fails ++ fcntl(fd, F_SETFD, FD_CLOEXEC); ++#endif ++ file_ = fopen(filename, "a+"); // Read and append a FILE*. ++ if (file_ == NULL) { // Man, we're screwed!, try to create new log file ++ close(fd); ++ } ++ fseek(file_, 0, SEEK_END); ++ file_length_ = bytes_since_flush_ = ftell(file_); ++ inited_ = true; ++ } ++ } ++ + // If there's no destination file, make one before outputting + if (file_ == NULL) { + // Try to rollover the log file every 32 log messages. The only time +@@ -1164,7 +1288,15 @@ void LogFileObject::Write(bool force_flush, + if (++rollover_attempt_ != kRolloverAttemptFrequency) return; + rollover_attempt_ = 0; + +- struct ::tm tm_time; ++ if (!inited_) { ++ CheckFileNumQuota(); ++ inited_ = true; ++ } else { ++ while (FLAGS_log_filenum_quota > 0 && file_list_.size() >= FLAGS_log_filenum_quota) { ++ unlink(file_list_.front().name.c_str()); ++ file_list_.pop_front(); ++ } ++ } + if (FLAGS_log_utc_time) { + gmtime_r(×tamp, &tm_time); + } else { +@@ -1176,13 +1308,19 @@ void LogFileObject::Write(bool force_flush, + time_pid_stream.fill('0'); + time_pid_stream << 1900+tm_time.tm_year + << setw(2) << 1+tm_time.tm_mon +- << setw(2) << tm_time.tm_mday +- << '-' +- << setw(2) << tm_time.tm_hour +- << setw(2) << tm_time.tm_min +- << setw(2) << tm_time.tm_sec +- << '.' +- << GetMainThreadPid(); ++ << setw(2) << tm_time.tm_mday; ++ ++ if ("hour" == FLAGS_log_split_method) { ++ time_pid_stream << setw(2) << tm_time.tm_hour; ++ } else if ("day" != FLAGS_log_split_method) { ++ time_pid_stream << '-' ++ << setw(2) << tm_time.tm_hour ++ << setw(2) << tm_time.tm_min ++ << setw(2) << tm_time.tm_sec; ++ } ++ ++ tm_time_ = tm_time; ++ + const string& time_pid_string = time_pid_stream.str(); + + if (base_filename_selected_) { +@@ -1216,9 +1354,7 @@ void LogFileObject::Write(bool force_flush, + // deadlock. Simply use a name like invalid-user. + if (uidname.empty()) uidname = "invalid-user"; + +- stripped_filename = stripped_filename+'.'+hostname+'.' +- +uidname+".log." +- +LogSeverityNames[severity_]+'.'; ++ stripped_filename = stripped_filename + "." + LogSeverityNames[severity_] + ".log."; + // We're going to (potentially) try to put logs in several different dirs + const vector & log_dirs = GetLoggingDirectories(); + +@@ -1243,36 +1379,6 @@ void LogFileObject::Write(bool force_flush, + } + } + +- // Write a header message into the log file +- ostringstream file_header_stream; +- file_header_stream.fill('0'); +- file_header_stream << "Log file created at: " +- << 1900+tm_time.tm_year << '/' +- << setw(2) << 1+tm_time.tm_mon << '/' +- << setw(2) << tm_time.tm_mday +- << ' ' +- << setw(2) << tm_time.tm_hour << ':' +- << setw(2) << tm_time.tm_min << ':' +- << setw(2) << tm_time.tm_sec << (FLAGS_log_utc_time ? " UTC\n" : "\n") +- << "Running on machine: " +- << LogDestination::hostname() << '\n'; +- +- if(!g_application_fingerprint.empty()) { +- file_header_stream << "Application fingerprint: " << g_application_fingerprint << '\n'; +- } +- const char* const date_time_format = FLAGS_log_year_in_prefix +- ? "yyyymmdd hh:mm:ss.uuuuuu" +- : "mmdd hh:mm:ss.uuuuuu"; +- file_header_stream << "Running duration (h:mm:ss): " +- << PrettyDuration(static_cast(WallTime_Now() - start_time_)) << '\n' +- << "Log line format: [IWEF]" << date_time_format << " " +- << "threadid file:line] msg" << '\n'; +- const string& file_header_string = file_header_stream.str(); +- +- const size_t header_len = file_header_string.size(); +- fwrite(file_header_string.data(), 1, header_len, file_); +- file_length_ += header_len; +- bytes_since_flush_ += header_len; + } + + // Write to LOG file \ No newline at end of file diff --git a/thirdparty/patches/opentelemetry-cpp-1.8.3.patch b/thirdparty/patches/opentelemetry-cpp-1.10.0.patch similarity index 93% rename from thirdparty/patches/opentelemetry-cpp-1.8.3.patch rename to thirdparty/patches/opentelemetry-cpp-1.10.0.patch index 534d6056f6e0ea..b0e7102a6d2504 100644 --- a/thirdparty/patches/opentelemetry-cpp-1.8.3.patch +++ b/thirdparty/patches/opentelemetry-cpp-1.10.0.patch @@ -1,21 +1,6 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index f4fa0649..a64e74bc 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -118,9 +118,8 @@ option(WITH_GSL - - option(WITH_ABSEIL "Whether to use Abseil for C++latest features" OFF) - -+set(OPENTELEMETRY_INSTALL_default ON) - if(NOT CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) -- set(OPENTELEMETRY_INSTALL_default ON) --else() - set(OPENTELEMETRY_INSTALL_default OFF) - endif() - option(OPENTELEMETRY_INSTALL "Whether to install opentelemetry targets" diff --git a/api/include/opentelemetry/common/threadlocal.h b/api/include/opentelemetry/common/threadlocal.h new file mode 100644 -index 00000000..23a39e14 +index 00000000..623a6454 --- /dev/null +++ b/api/include/opentelemetry/common/threadlocal.h @@ -0,0 +1,123 @@ @@ -26,12 +11,12 @@ index 00000000..23a39e14 + +#include "opentelemetry/version.h" + -+// ++ +// GCC can be told that a certain branch is not likely to be taken (for +// instance, a CHECK failure), and use that information in static analysis. +// Giving it this information can help it optimize for the common case in +// the absence of better information (ie. -fprofile-arcs). -+// ++ +#define LIKELY(expr) __builtin_expect(!!(expr), 1) +#define UNLIKELY(expr) __builtin_expect(!!(expr), 0) + @@ -144,27 +129,27 @@ index 00000000..23a39e14 +OPENTELEMETRY_END_NAMESPACE \ No newline at end of file diff --git a/api/include/opentelemetry/context/runtime_context.h b/api/include/opentelemetry/context/runtime_context.h -index bec96a9a..2349a29c 100644 +index 2cd5b0ff..9e3c5c47 100644 --- a/api/include/opentelemetry/context/runtime_context.h +++ b/api/include/opentelemetry/context/runtime_context.h @@ -5,6 +5,7 @@ - + #include "opentelemetry/common/macros.h" #include "opentelemetry/context/context.h" +#include "opentelemetry/common/threadlocal.h" - - OPENTELEMETRY_BEGIN_NAMESPACE - namespace context -@@ -189,7 +190,7 @@ class ThreadLocalContextStorage : public RuntimeContextStorage + #include "opentelemetry/nostd/shared_ptr.h" + #include "opentelemetry/nostd/string_view.h" + #include "opentelemetry/nostd/unique_ptr.h" +@@ -193,7 +194,7 @@ class ThreadLocalContextStorage : public RuntimeContextStorage ThreadLocalContextStorage() noexcept = default; - + // Return the current context. - Context GetCurrent() noexcept override { return GetStack().Top(); } + Context GetCurrent() noexcept override { return GetStack()->Top(); } - + // Resets the context to the value previous to the passed in token. This will // also detach all child contexts of the passed in token. -@@ -197,23 +198,23 @@ class ThreadLocalContextStorage : public RuntimeContextStorage +@@ -201,23 +202,23 @@ class ThreadLocalContextStorage : public RuntimeContextStorage bool Detach(Token &token) noexcept override { // In most cases, the context to be detached is on the top of the stack. @@ -175,26 +160,26 @@ index bec96a9a..2349a29c 100644 + GetStack()->Pop(); return true; } - + - if (!GetStack().Contains(token)) + if (!GetStack()->Contains(token)) { return false; } - + - while (!(token == GetStack().Top())) + while (!(token == GetStack()->Top())) { - GetStack().Pop(); + GetStack()->Pop(); } - + - GetStack().Pop(); + GetStack()->Pop(); - + return true; } -@@ -222,14 +223,14 @@ class ThreadLocalContextStorage : public RuntimeContextStorage +@@ -226,14 +227,14 @@ class ThreadLocalContextStorage : public RuntimeContextStorage // that can be used to reset to the previous Context. nostd::unique_ptr Attach(const Context &context) noexcept override { @@ -202,19 +187,19 @@ index bec96a9a..2349a29c 100644 + GetStack()->Push(context); return CreateToken(context); } - + -private: // A nested class to store the attached contexts in a stack. class Stack { + public: friend class ThreadLocalContextStorage; - + Stack() noexcept : size_(0), capacity_(0), base_(nullptr) {} -@@ -316,9 +317,10 @@ class ThreadLocalContextStorage : public RuntimeContextStorage +@@ -320,9 +321,10 @@ class ThreadLocalContextStorage : public RuntimeContextStorage Context *base_; }; - + - OPENTELEMETRY_API_SINGLETON Stack &GetStack() + OPENTELEMETRY_API_SINGLETON Stack *GetStack() { @@ -231,20 +216,32 @@ index 2e8f3bb6..224e6463 100644 @@ -1,7 +1,7 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 - + -add_library(opentelemetry_sdk INTERFACE) +add_library(opentelemetry_sdk INTERFACE ../api/include/opentelemetry/common/threadlocal.h) target_include_directories( opentelemetry_sdk INTERFACE "$" +diff --git a/sdk/include/opentelemetry/sdk/metrics/data/circular_buffer.h b/sdk/include/opentelemetry/sdk/metrics/data/circular_buffer.h +index 1608a3a0..6af2bd06 100644 +--- a/sdk/include/opentelemetry/sdk/metrics/data/circular_buffer.h ++++ b/sdk/include/opentelemetry/sdk/metrics/data/circular_buffer.h +@@ -7,6 +7,7 @@ + + #include + #include ++#include + + OPENTELEMETRY_BEGIN_NAMESPACE + namespace sdk diff --git a/sdk/src/common/CMakeLists.txt b/sdk/src/common/CMakeLists.txt -index b23a93e8..1cc0e0f2 100644 +index 7bb645f5..041d8ede 100644 --- a/sdk/src/common/CMakeLists.txt +++ b/sdk/src/common/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 - + -set(COMMON_SRCS random.cc core.cc global_log_handler.cc env_variables.cc) +set(COMMON_SRCS random.cc core.cc global_log_handler.cc env_variables.cc threadlocal.cc) if(WIN32) @@ -256,26 +253,26 @@ index 77b88cfa..dc71f9c1 100644 +++ b/sdk/src/common/random.cc @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 - + #include "src/common/random.h" +#include "opentelemetry/common/threadlocal.h" #include "src/common/platform/fork.h" - + #include @@ -29,33 +30,37 @@ class TlsRandomNumberGenerator platform::AtFork(nullptr, nullptr, OnFork); } - + - static FastRandomNumberGenerator &engine() noexcept { return engine_; } + static FastRandomNumberGenerator *engine() noexcept { return engine_; } - + private: - static thread_local FastRandomNumberGenerator engine_; + // static thread_local FastRandomNumberGenerator engine_; + DECLARE_STATIC_THREAD_LOCAL_TELEMETRY(FastRandomNumberGenerator, engine_); - + static void OnFork() noexcept { Seed(); } - + static void Seed() noexcept { + INIT_STATIC_THREAD_LOCAL_TELEMETRY(FastRandomNumberGenerator, engine_); @@ -285,12 +282,12 @@ index 77b88cfa..dc71f9c1 100644 + engine_->seed(seed_seq); } }; - + -thread_local FastRandomNumberGenerator TlsRandomNumberGenerator::engine_{}; +// thread_local FastRandomNumberGenerator TlsRandomNumberGenerator::engine_{}; +DEFINE_STATIC_THREAD_LOCAL_TELEMETRY(FastRandomNumberGenerator, TlsRandomNumberGenerator, engine_); } // namespace - + -FastRandomNumberGenerator &Random::GetRandomNumberGenerator() noexcept +FastRandomNumberGenerator *Random::GetRandomNumberGenerator() noexcept { @@ -299,13 +296,13 @@ index 77b88cfa..dc71f9c1 100644 + BLOCK_STATIC_THREAD_LOCAL_TELEMTRY(TlsRandomNumberGenerator, random_number_generator); return TlsRandomNumberGenerator::engine(); } - + uint64_t Random::GenerateRandom64() noexcept { - return GetRandomNumberGenerator()(); + return GetRandomNumberGenerator()->operator()(); } - + void Random::GenerateRandomBuffer(opentelemetry::nostd::span buffer) noexcept diff --git a/sdk/src/common/random.h b/sdk/src/common/random.h index ecd6dabc..1aaa2204 100644 @@ -409,4 +406,4 @@ index 00000000..ec2038b4 +} +} // namespace internal_threadlocal +OPENTELEMETRY_END_NAMESPACE -\ No newline at end of file +\ No newline at end of file \ No newline at end of file diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 435434927fe209..8fdafe184b44e0 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -73,10 +73,11 @@ THRIFT_SOURCE=thrift-0.16.0 THRIFT_MD5SUM="44cf1b54b4ec1890576c85804acfa637" # protobuf -PROTOBUF_DOWNLOAD="https://github.com/google/protobuf/archive/v3.15.0.tar.gz" -PROTOBUF_NAME=protobuf-3.15.0.tar.gz -PROTOBUF_SOURCE=protobuf-3.15.0 -PROTOBUF_MD5SUM="3fb6c41f7b3c621424543f43d54baa42" +# brpc is not yet compatible with protobuf >= 22 +PROTOBUF_DOWNLOAD="https://github.com/protocolbuffers/protobuf/releases/download/v21.11/protobuf-all-21.11.tar.gz" +PROTOBUF_NAME="protobuf-all-21.11.tar.gz" +PROTOBUF_SOURCE=protobuf-21.11 +PROTOBUF_MD5SUM="b3b104f0374802e1add5d5d7a5a845ac" # gflags GFLAGS_DOWNLOAD="https://github.com/gflags/gflags/archive/v2.2.2.tar.gz" @@ -85,10 +86,10 @@ GFLAGS_SOURCE=gflags-2.2.2 GFLAGS_MD5SUM="1a865b93bacfa963201af3f75b7bd64c" # glog -GLOG_DOWNLOAD="https://github.com/google/glog/archive/v0.4.0.tar.gz" -GLOG_NAME=glog-0.4.0.tar.gz -GLOG_SOURCE=glog-0.4.0 -GLOG_MD5SUM="0daea8785e6df922d7887755c3d100d0" +GLOG_DOWNLOAD="https://github.com/google/glog/archive/refs/tags/v0.6.0.tar.gz" +GLOG_NAME="glog-v0.6.0.tar.gz" +GLOG_SOURCE=glog-0.6.0 +GLOG_MD5SUM="c98a6068bc9b8ad9cebaca625ca73aa2" # gtest GTEST_DOWNLOAD="https://github.com/google/googletest/archive/release-1.11.0.tar.gz" @@ -115,10 +116,10 @@ ZLIB_SOURCE=zlib-1.2.11 ZLIB_MD5SUM="1c9f62f0778697a09d36121ead88e08e" # lz4 -LZ4_DOWNLOAD="https://github.com/lz4/lz4/archive/v1.9.3.tar.gz" -LZ4_NAME=lz4-1.9.3.tar.gz -LZ4_SOURCE=lz4-1.9.3 -LZ4_MD5SUM="3a1ab1684e14fc1afc66228ce61b2db3" +LZ4_DOWNLOAD="https://github.com/lz4/lz4/archive/v1.9.4.tar.gz" +LZ4_NAME=lz4-1.9.4.tar.gz +LZ4_SOURCE=lz4-1.9.4 +LZ4_MD5SUM="e9286adb64040071c5e23498bf753261" # bzip BZIP_DOWNLOAD="https://fossies.org/linux/misc/bzip2-1.0.8.tar.gz" @@ -127,7 +128,7 @@ BZIP_SOURCE=bzip2-1.0.8 BZIP_MD5SUM="67e051268d0c475ea773822f7500d0e5" # lzo2 -LZO2_DOWNLOAD="http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz" +LZO2_DOWNLOAD="https://fossies.org/linux/misc/lzo-2.10.tar.gz" LZO2_NAME=lzo-2.10.tar.gz LZO2_SOURCE=lzo-2.10 LZO2_MD5SUM="39d3f3f9c55c87b1e5d6888e1420f4b5" @@ -139,10 +140,10 @@ RAPIDJSON_SOURCE=rapidjson-1a803826f1197b5e30703afe4b9c0e7dd48074f5 RAPIDJSON_MD5SUM="f2212a77e055a15501477f1e390007ea" # curl -CURL_DOWNLOAD="https://curl.se/download/curl-7.79.0.tar.gz" -CURL_NAME=curl-7.79.0.tar.gz -CURL_SOURCE=curl-7.79.0 -CURL_MD5SUM="b40e4dc4bbc9e109c330556cd58c8ec8" +CURL_DOWNLOAD="https://curl.se/download/curl-8.2.1.tar.gz" +CURL_NAME="curl-8.2.1.tar.gz" +CURL_SOURCE=curl-8.2.1 +CURL_MD5SUM="b25588a43556068be05e1624e0e74d41" # RE2 RE2_DOWNLOAD="https://github.com/google/re2/archive/2021-02-02.tar.gz" @@ -197,10 +198,10 @@ LEVELDB_SOURCE=leveldb-1.23 LEVELDB_MD5SUM="afbde776fb8760312009963f09a586c7" # brpc -BRPC_DOWNLOAD="https://github.com/apache/brpc/archive/refs/tags/1.4.0.tar.gz" -BRPC_NAME="brpc-1.4.0.tar.gz" -BRPC_SOURCE="brpc-1.4.0" -BRPC_MD5SUM="6af9d50822c33a3abc56a1ec0af0e0bc" +BRPC_DOWNLOAD="https://github.com/apache/brpc/archive/refs/tags/1.6.0.tar.gz" +BRPC_NAME="brpc-1.6.0.tar.gz" +BRPC_SOURCE="brpc-1.6.0" +BRPC_MD5SUM="0d37cea25bd006e89806f461ef7e39ba" # rocksdb ROCKSDB_DOWNLOAD="https://github.com/facebook/rocksdb/archive/v5.14.2.tar.gz" @@ -221,10 +222,10 @@ LIBRDKAFKA_SOURCE=librdkafka-1.8.2 LIBRDKAFKA_MD5SUM="0abec0888d10c9553cdcbcbf9172d558" # zstd -ZSTD_DOWNLOAD="https://github.com/facebook/zstd/releases/download/v1.5.2/zstd-1.5.2.tar.gz" -ZSTD_NAME=zstd-1.5.2.tar.gz -ZSTD_SOURCE=zstd-1.5.2 -ZSTD_MD5SUM="072b10f71f5820c24761a65f31f43e73" +ZSTD_DOWNLOAD="https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz" +ZSTD_NAME=zstd-1.5.5.tar.gz +ZSTD_SOURCE=zstd-1.5.5 +ZSTD_MD5SUM="63251602329a106220e0a5ad26ba656f" # brotli BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz" @@ -238,17 +239,30 @@ FLATBUFFERS_NAME=flatbuffers-2.0.0.tar.gz FLATBUFFERS_SOURCE=flatbuffers-2.0.0 FLATBUFFERS_MD5SUM="a27992324c3cbf86dd888268a23d17bd" +# c-ares +CARES_DOWNLOAD="https://github.com/c-ares/c-ares/releases/download/cares-1_19_1/c-ares-1.19.1.tar.gz" +CARES_NAME="c-ares-1.19.1.tar.gz" +CARES_SOURCE=c-ares-1.19.1 +CARES_MD5SUM="dafc5825a92dc907e144570e4e75a908" + +# grpc +# grpc v1.55 and above require protobuf >= 22 +GRPC_DOWNLOAD="https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz" +GRPC_NAME="grpc-v1.54.3.tar.gz" +GRPC_SOURCE=grpc-1.54.3 +GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de" + # arrow -ARROW_DOWNLOAD="https://archive.apache.org/dist/arrow/arrow-7.0.0/apache-arrow-7.0.0.tar.gz" -ARROW_NAME="apache-arrow-7.0.0.tar.gz" -ARROW_SOURCE="apache-arrow-7.0.0" -ARROW_MD5SUM="316ade159901646849b3b4760fa52816" +ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-13.0.0.tar.gz" +ARROW_NAME="apache-arrow-13.0.0.tar.gz" +ARROW_SOURCE="arrow-apache-arrow-13.0.0" +ARROW_MD5SUM="8ec1ec6a119514bcaea1cf7aabc9df1f" # Abseil -ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz" -ABSEIL_NAME=abseil-cpp-20220623.1.tar.gz -ABSEIL_SOURCE=abseil-cpp-20220623.1 -ABSEIL_MD5SUM="2aea7c1171c4c280f755de170295afd6" +ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/archive/refs/tags/20230125.3.tar.gz" +ABSEIL_NAME="abseil-cpp-20230125.3.tar.gz" +ABSEIL_SOURCE=abseil-cpp-20230125.3 +ABSEIL_MD5SUM="9b6dae642c4bd92f007ab2c148bc0498" # S2 S2_DOWNLOAD="https://github.com/google/s2geometry/archive/refs/tags/v0.10.0.tar.gz" @@ -281,16 +295,16 @@ PARALLEL_HASHMAP_SOURCE="parallel-hashmap-1.3.8" PARALLEL_HASHMAP_MD5SUM="1b8130d0b4f656257ef654699bfbf941" # orc -ORC_DOWNLOAD="https://archive.apache.org/dist/orc/orc-1.7.2/orc-1.7.2.tar.gz" -ORC_NAME=orc-1.7.2.tar.gz -ORC_SOURCE=orc-1.7.2 -ORC_MD5SUM="6cab37935eacdec7d078d327746a8578" +ORC_DOWNLOAD="https://archive.apache.org/dist/orc/orc-1.9.0/orc-1.9.0.tar.gz" +ORC_NAME="orc-1.9.0.tar.gz" +ORC_SOURCE=orc-1.9.0 +ORC_MD5SUM="5dc1c91c4867e4519aab531ffc30fab7" # jemalloc for arrow -JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2" -JEMALLOC_ARROW_NAME="jemalloc-5.2.1.tar.bz2" -JEMALLOC_ARROW_SOURCE="jemalloc-5.2.1" -JEMALLOC_ARROW_MD5SUM="3d41fbf006e6ebffd489bdb304d009ae" +JEMALLOC_ARROW_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2" +JEMALLOC_ARROW_NAME="jemalloc-5.3.0.tar.bz2" +JEMALLOC_ARROW_SOURCE="jemalloc-5.3.0" +JEMALLOC_ARROW_MD5SUM="09a8328574dab22a7df848eae6dbbf53" # jemalloc for doris JEMALLOC_DORIS_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2" @@ -393,16 +407,16 @@ PDQSORT_MD5SUM="af28f79d5d7d7a5486f54d9f1244c2b5" # benchmark BENCHMARK_DOWNLOAD="https://github.com/google/benchmark/archive/refs/tags/v1.8.0.tar.gz" -BENCHMARK_NAME=v1.8.0.tar.gz +BENCHMARK_NAME=benchmark-v1.8.0.tar.gz BENCHMARK_SOURCE=benchmark-1.8.0 BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817" # xsimd -# for arrow-7.0.0, if arrow upgrade, this version may also need to be changed -XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz" -XSIMD_NAME=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz -XSIMD_SOURCE=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2 -XSIMD_MD5SUM="d024855f71c0a2837a6918c0f8f66245" +# for arrow-13.0.0, if arrow upgrade, this version may also need to be changed +XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/9.0.1.tar.gz" +XSIMD_NAME="xsimd-9.0.1.tar.gz" +XSIMD_SOURCE=xsimd-9.0.1 +XSIMD_MD5SUM="59f38fe3364acd7ed137771258812d6c" # simdjson SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.0.1.tar.gz" @@ -417,16 +431,16 @@ NLOHMANN_JSON_SOURCE=json-3.10.1 NLOHMANN_JSON_MD5SUM="7b369d567afc0dffdcf5800fd9abb836" # opentelemetry-proto -OPENTELEMETRY_PROTO_DOWNLOAD="https://github.com/open-telemetry/opentelemetry-proto/archive/refs/tags/v0.19.0.tar.gz" -OPENTELEMETRY_PROTO_NAME=opentelemetry-proto-0.19.0.tar.gz -OPENTELEMETRY_PROTO_SOURCE=opentelemetry-proto-0.19.0 -OPENTELEMETRY_PROTO_MD5SUM="8a5fb1ba721341994a5e7cae8b38bcc6" +OPENTELEMETRY_PROTO_DOWNLOAD="https://github.com/open-telemetry/opentelemetry-proto/archive/refs/tags/v1.0.0.tar.gz" +OPENTELEMETRY_PROTO_NAME="opentelemetry-proto-v1.0.0.tar.gz" +OPENTELEMETRY_PROTO_SOURCE=opentelemetry-proto-1.0.0 +OPENTELEMETRY_PROTO_MD5SUM="8c7495a0dceea7cfdbdbcd53b07436dc" # opentelemetry -OPENTELEMETRY_DOWNLOAD="https://github.com/open-telemetry/opentelemetry-cpp/archive/refs/tags/v1.8.3.tar.gz" -OPENTELEMETRY_NAME=opentelemetry-cpp-1.8.3.tar.gz -OPENTELEMETRY_SOURCE=opentelemetry-cpp-1.8.3 -OPENTELEMETRY_MD5SUM="a051dbbced2b7eb7a7227bdf5b3b9a10" +OPENTELEMETRY_DOWNLOAD="https://github.com/open-telemetry/opentelemetry-cpp/archive/refs/tags/v1.10.0.tar.gz" +OPENTELEMETRY_NAME="opentelemetry-cpp-v1.10.0.tar.gz" +OPENTELEMETRY_SOURCE=opentelemetry-cpp-1.10.0 +OPENTELEMETRY_MD5SUM="89169762241b2f5142b728c775173283" # libbacktrace LIBBACKTRACE_DOWNLOAD="https://codeload.github.com/ianlancetaylor/libbacktrace/zip/2446c66076480ce07a6bd868badcbceb3eeecc2e" @@ -503,6 +517,8 @@ export TP_ARCHIVES=( 'FMT' 'PARALLEL_HASHMAP' 'ORC' + 'CARES' + 'GRPC' 'JEMALLOC_ARROW' 'JEMALLOC_DORIS' 'LIBUNWIND'