From 9d6598108c4fd93be89f8f5becaa7cb66f929fe7 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 13 Sep 2022 08:40:20 +0200 Subject: [PATCH] ARROW-17545: [C++][CI] Mandate C++17 instead of C++11 (#13991) This PR switches our build system to require C++17 instead of C++11. Because the conda packaging jobs are out of sync with the conda-forge files, the Windows conda packaging jobs are broken with this change. The related task (sync conda packaging files with conda-forge) is tracked in ARROW-17635. Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- .env | 4 +- .github/workflows/r.yml | 12 +--- c_glib/meson.build | 2 +- ci/conan/all/test_package/CMakeLists.txt | 2 +- ci/docker/linux-r.dockerfile | 2 +- ci/scripts/cpp_build.sh | 2 +- ci/scripts/r_docker_configure.sh | 4 +- ci/scripts/r_test.sh | 20 +++++-- cpp/CMakeLists.txt | 6 +- cpp/build-support/update-flatbuffers.sh | 3 - cpp/cmake_modules/SetupCxxFlags.cmake | 18 +++--- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- cpp/examples/minimal_build/CMakeLists.txt | 4 +- .../parquet/parquet_arrow/CMakeLists.txt | 6 +- cpp/src/arrow/CMakeLists.txt | 2 + cpp/src/arrow/dataset/file_csv.h | 3 +- cpp/src/arrow/dataset/file_ipc.h | 3 +- cpp/src/arrow/dataset/file_parquet.h | 3 +- cpp/src/arrow/memory_pool.cc | 4 ++ cpp/src/arrow/util/aligned_storage.h | 19 +++++++ cpp/src/arrow/util/int_util_test.cc | 55 +++++++++++++++++++ cpp/src/arrow/util/launder.h | 2 +- cpp/src/gandiva/CMakeLists.txt | 13 ----- cpp/src/gandiva/precompiled/CMakeLists.txt | 4 +- cpp/src/gandiva/symbols.map | 4 +- cpp/src/parquet/symbols.map | 4 +- dev/archery/archery/crossbow/core.py | 3 +- dev/release/verify-apt.sh | 2 +- dev/release/verify-yum.sh | 21 ++++++- ...ion10.2numpy1.18python3.7.____cpython.yaml | 4 +- ...ion10.2numpy1.18python3.8.____cpython.yaml | 4 +- ...ion10.2numpy1.19python3.9.____cpython.yaml | 4 +- ...on10.2numpy1.21python3.10.____cpython.yaml | 4 +- ...ionNonenumpy1.18python3.7.____cpython.yaml | 4 +- ...ionNonenumpy1.18python3.8.____cpython.yaml | 4 +- ...ionNonenumpy1.19python3.9.____cpython.yaml | 4 +- ...onNonenumpy1.21python3.10.____cpython.yaml | 4 +- .../conda-recipes/arrow-cpp/bld-arrow.bat | 4 +- .../conda-recipes/arrow-cpp/bld-pyarrow.bat | 2 +- .../conda-recipes/arrow-cpp/build-arrow.sh | 18 +++--- .../conda-recipes/arrow-cpp/build-pyarrow.sh | 9 ++- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 7 ++- dev/tasks/conda-recipes/azure.win.yml | 24 +------- dev/tasks/conda-recipes/r-arrow/build.sh | 6 ++ dev/tasks/conda-recipes/r-arrow/configure.win | 2 +- dev/tasks/homebrew-formulae/apache-arrow.rb | 2 +- .../autobrew/apache-arrow-static.rb | 2 +- .../autobrew/apache-arrow.rb | 2 +- .../apache-arrow/yum/arrow.spec.in | 10 +++- .../apache-arrow/yum/centos-7/Dockerfile | 11 +++- dev/tasks/macros.jinja | 3 +- dev/tasks/r/azure.linux.yml | 2 +- dev/tasks/r/github.packages.yml | 6 +- dev/tasks/tasks.yml | 14 ++--- dev/tasks/verify-rc/github.macos.amd64.yml | 4 ++ docs/source/cpp/conventions.rst | 3 +- docs/source/developers/cpp/building.rst | 4 +- docs/source/java/cdata.rst | 4 +- docs/source/python/integration/extending.rst | 11 +--- java/CMakeLists.txt | 2 +- java/c/README.md | 2 +- matlab/CMakeLists.txt | 2 +- matlab/README.md | 2 +- python/CMakeLists.txt | 10 ++-- python/pyarrow/plasma.py | 2 +- python/pyarrow/src/CMakeLists.txt | 9 ++- python/pyarrow/tests/test_cython.py | 4 +- python/setup.py | 2 + r/DESCRIPTION | 2 +- r/configure | 13 ++--- r/src/Makevars.in | 2 +- r/tools/nixlibs.R | 8 +-- ruby/red-arrow/ext/arrow/extconf.rb | 2 + 73 files changed, 284 insertions(+), 189 deletions(-) diff --git a/.env b/.env index 8ae036b6b5be9..761642506f1af 100644 --- a/.env +++ b/.env @@ -86,8 +86,8 @@ ARROW_R_DEV=TRUE R_PRUNE_DEPS=FALSE TZ=UTC -# -1 does not attempt to install a devtoolset version, any positive integer will install devtoolset-n -DEVTOOLSET_VERSION=-1 +# Any non-empty string will install devtoolset-${DEVTOOLSET_VERSION} +DEVTOOLSET_VERSION= # Used through docker-compose.yml and serves as the default version for the # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index f1bafbef3b713..1b0fdd3d9d6c4 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -119,12 +119,13 @@ jobs: fail-fast: false matrix: config: - - { org: "rstudio", image: "r-base", tag: "4.0-centos7" } - - { org: "rhub", image: "debian-gcc-devel", tag: "latest" } + - { org: "rstudio", image: "r-base", tag: "4.0-centos7", devtoolset: "8" } + - { org: "rhub", image: "debian-gcc-devel", tag: "latest", devtoolset: "" } env: R_ORG: ${{ matrix.config.org }} R_IMAGE: ${{ matrix.config.image }} R_TAG: ${{ matrix.config.tag }} + DEVTOOLSET_VERSION: ${{ matrix.config.devtoolset }} steps: - name: Checkout Arrow uses: actions/checkout@v3 @@ -184,11 +185,6 @@ jobs: run: | ci/scripts/ccache_setup.sh echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV - # We must enable actions/cache before r-lib/actions/setup-r to ensure - # using system tar instead of tar provided by Rtools. - # We can use tar provided by Rtools when we drop support for Rtools 3.5. - # Because Rtools 4.0 or later has zstd. actions/cache requires zstd - # when tar is GNU tar. - name: Cache ccache uses: actions/cache@v2 with: @@ -268,8 +264,6 @@ jobs: with: r-version: ${{ matrix.config.rversion }} rtools-version: ${{ matrix.config.rtools }} - # RSPM keeps install times short for 3.6 - use-public-rspm: true Ncpus: 2 - uses: r-lib/actions/setup-r-dependencies@v2 env: diff --git a/c_glib/meson.build b/c_glib/meson.build index 85d3a75d42381..8b69cfdb16752 100644 --- a/c_glib/meson.build +++ b/c_glib/meson.build @@ -21,7 +21,7 @@ project('arrow-glib', 'c', 'cpp', license: 'Apache-2.0', default_options: [ 'c_std=c99', - 'cpp_std=c++11', + 'cpp_std=c++17', ]) version = '10.0.0-SNAPSHOT' diff --git a/ci/conan/all/test_package/CMakeLists.txt b/ci/conan/all/test_package/CMakeLists.txt index 0df89423c14ef..4ea8b3605d1dc 100644 --- a/ci/conan/all/test_package/CMakeLists.txt +++ b/ci/conan/all/test_package/CMakeLists.txt @@ -30,5 +30,5 @@ find_package(Arrow REQUIRED) add_executable(${PROJECT_NAME} test_package.cpp) target_link_libraries(${PROJECT_NAME} arrow::arrow) -target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_11) +target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_17) target_compile_definitions(${PROJECT_NAME} PRIVATE WITH_JEMALLOC) diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile index 804fb09f09cc2..96420fcfc69a4 100644 --- a/ci/docker/linux-r.dockerfile +++ b/ci/docker/linux-r.dockerfile @@ -27,7 +27,7 @@ ENV R_BIN=${r_bin} ARG r_dev=FALSE ENV ARROW_R_DEV=${r_dev} -ARG devtoolset_version=-1 +ARG devtoolset_version= ENV DEVTOOLSET_VERSION=${devtoolset_version} ARG r_prune_deps=FALSE diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index c7d200697820a..68437938b5ef7 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -142,7 +142,7 @@ cmake \ -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE:-OFF} \ -DCMAKE_C_FLAGS="${CFLAGS:-}" \ -DCMAKE_CXX_FLAGS="${CXXFLAGS:-}" \ - -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD:-11}" \ + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD:-17}" \ -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \ -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index f7795c3949234..79bf55550e147 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -57,7 +57,7 @@ if [ ${R_CUSTOM_CCACHE} = "true" ]; then CCACHE=ccache CC=\$(CCACHE) gcc\$(VER) CXX=\$(CCACHE) g++\$(VER) -CXX11=\$(CCACHE) g++\$(VER)" >> ~/.R/Makevars +CXX17=\$(CCACHE) g++\$(VER)" >> ~/.R/Makevars mkdir -p ~/.ccache/ echo "max_size = 5.0G @@ -69,7 +69,7 @@ fi # Special hacking to try to reproduce quirks on centos using non-default build # tooling. -if [[ "$DEVTOOLSET_VERSION" -gt 0 ]]; then +if [[ -n "$DEVTOOLSET_VERSION" ]]; then $PACKAGE_MANAGER install -y centos-release-scl $PACKAGE_MANAGER install -y "devtoolset-$DEVTOOLSET_VERSION" fi diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index 0328df2384b56..519144ab4c5c3 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -26,8 +26,21 @@ pushd ${source_dir} printenv +if [[ -n "$DEVTOOLSET_VERSION" ]]; then + # enable the devtoolset version to use it + source /opt/rh/devtoolset-$DEVTOOLSET_VERSION/enable + + # Build images which require the devtoolset don't have CXX17 variables + # set as the system compiler doesn't support C++17 + mkdir -p ~/.R + echo "CC = $(which gcc) -fPIC" >> ~/.R/Makevars + echo "CXX17 = $(which g++) -fPIC" >> ~/.R/Makevars + echo "CXX17STD = -std=c++17" >> ~/.R/Makevars + echo "CXX17FLAGS = ${CXX11FLAGS}" >> ~/.R/Makevars +fi + # Run the nixlibs.R test suite, which is not included in the installed package - ${R_BIN} -e 'setwd("tools"); testthat::test_dir(".")' +${R_BIN} -e 'setwd("tools"); testthat::test_dir(".")' # Before release, we always copy the relevant parts of the cpp source into the # package. In some CI checks, we will use this version of the source: @@ -77,11 +90,6 @@ export ARROW_DEBUG_MEMORY_POOL=trap export TEXMFCONFIG=/tmp/texmf-config export TEXMFVAR=/tmp/texmf-var -if [[ "$DEVTOOLSET_VERSION" -gt 0 ]]; then - # enable the devtoolset version to use it - source /opt/rh/devtoolset-$DEVTOOLSET_VERSION/enable -fi - # Make sure we aren't writing to the home dir (CRAN _hates_ this but there is no official check) BEFORE=$(ls -alh ~/) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4dbdd2353fd48..1a728bd787060 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -561,10 +561,10 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARROW_CXXFLAGS}") # C++ specific flags. set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CXX_COMMON_FLAGS} ${ARROW_CXXFLAGS}") -# Remove --std=c++11 to avoid errors from C compilers -string(REPLACE "-std=c++11" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) +# Remove --std=c++17 to avoid errors from C compilers +string(REPLACE "-std=c++17" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) -# Add C++-only flags, like -std=c++11 +# Add C++-only flags, like -std=c++17 set(CMAKE_CXX_FLAGS "${CXX_ONLY_FLAGS} ${CMAKE_CXX_FLAGS}") # ASAN / TSAN / UBSAN diff --git a/cpp/build-support/update-flatbuffers.sh b/cpp/build-support/update-flatbuffers.sh index b1116a1cbf703..52da752068a8a 100755 --- a/cpp/build-support/update-flatbuffers.sh +++ b/cpp/build-support/update-flatbuffers.sh @@ -33,9 +33,6 @@ OUT_DIR="$SOURCE_DIR/generated" FILES=($(find $FORMAT_DIR -name '*.fbs')) FILES+=("$SOURCE_DIR/arrow/ipc/feather.fbs") -# add compute ir files -FILES+=($(find "$TOP/experimental/computeir" -name '*.fbs')) - $FLATC --cpp --cpp-std c++11 \ --scoped-enums \ -o "$OUT_DIR" \ diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index bc503145da4ae..a9e23be538614 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -122,12 +122,14 @@ if(NOT DEFINED CMAKE_C_STANDARD) set(CMAKE_C_STANDARD 11) endif() -# This ensures that things like c++11 get passed correctly +# This ensures that things like c++17 get passed correctly if(NOT DEFINED CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD 17) +elseif(${CMAKE_CXX_STANDARD} VERSION_LESS 17) + message(FATAL_ERROR "Cannot set a CMAKE_CXX_STANDARD smaller than 17") endif() -# We require a C++11 compliant compiler +# We require a C++17 compliant compiler set(CMAKE_CXX_STANDARD_REQUIRED ON) # ARROW-6848: Do not use GNU (or other CXX) extensions @@ -440,11 +442,11 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STRE # Don't complain about optimization passes that were not possible set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-pass-failed") - if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") - # Depending on the default OSX_DEPLOYMENT_TARGET (< 10.9), libstdc++ may be - # the default standard library which does not support C++11. libc++ is the - # default from 10.9 onward. - set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++") + # Avoid clang / libc++ error about C++17 aligned allocation on macOS. + # See https://chromium.googlesource.com/chromium/src/+/eee44569858fc650b635779c4e34be5cb0c73186%5E%21/#F0 + # for details. + if(APPLE) + set(CXX_ONLY_FLAGS "${CXX_ONLY_FLAGS} -fno-aligned-new") endif() endif() diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 515cdfe8ef494..3f7129ce553fa 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2088,7 +2088,7 @@ macro(build_benchmark) endif() if(NOT MSVC) - set(GBENCHMARK_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -std=c++11") + set(GBENCHMARK_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -std=c++17") endif() if(APPLE AND (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID diff --git a/cpp/examples/minimal_build/CMakeLists.txt b/cpp/examples/minimal_build/CMakeLists.txt index 6f4d9cefab4d6..b98f725a4a4c6 100644 --- a/cpp/examples/minimal_build/CMakeLists.txt +++ b/cpp/examples/minimal_build/CMakeLists.txt @@ -24,10 +24,10 @@ option(ARROW_LINK_SHARED "Link to the Arrow shared library" ON) find_package(Arrow REQUIRED) if(NOT DEFINED CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD 17) endif() -# We require a C++11 compliant compiler +# We require a C++17 compliant compiler set(CMAKE_CXX_STANDARD_REQUIRED ON) if(NOT DEFINED CMAKE_BUILD_TYPE) diff --git a/cpp/examples/parquet/parquet_arrow/CMakeLists.txt b/cpp/examples/parquet/parquet_arrow/CMakeLists.txt index 63f009a8f6272..84f9d16e40883 100644 --- a/cpp/examples/parquet/parquet_arrow/CMakeLists.txt +++ b/cpp/examples/parquet/parquet_arrow/CMakeLists.txt @@ -26,12 +26,12 @@ include(GNUInstallDirs) option(PARQUET_LINK_SHARED "Link to the Parquet shared library" ON) -# This ensures that things like gnu++11 get passed correctly +# This ensures that things like -std=gnu++... get passed correctly if(NOT DEFINED CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD 17) endif() -# We require a C++11 compliant compiler +# We require a C++17 compliant compiler set(CMAKE_CXX_STANDARD_REQUIRED ON) # Look for installed packages the system diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 2cae7a731f9d7..88d72b11832ae 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -636,6 +636,8 @@ endif() foreach(LIB_TARGET ${ARROW_LIBRARIES}) target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_EXPORTING) + # C++17 is required to compile against Arrow C++ headers and libraries + target_compile_features(${LIB_TARGET} PUBLIC cxx_std_17) endforeach() if(ARROW_WITH_BACKTRACE) diff --git a/cpp/src/arrow/dataset/file_csv.h b/cpp/src/arrow/dataset/file_csv.h index 84bcf94abe3bf..e58ed87b42720 100644 --- a/cpp/src/arrow/dataset/file_csv.h +++ b/cpp/src/arrow/dataset/file_csv.h @@ -98,7 +98,8 @@ class ARROW_DS_EXPORT CsvFileWriteOptions : public FileWriteOptions { std::shared_ptr write_options; protected: - using FileWriteOptions::FileWriteOptions; + explicit CsvFileWriteOptions(std::shared_ptr format) + : FileWriteOptions(std::move(format)) {} friend class CsvFileFormat; }; diff --git a/cpp/src/arrow/dataset/file_ipc.h b/cpp/src/arrow/dataset/file_ipc.h index 29ce6be61d653..6dc40da1faca4 100644 --- a/cpp/src/arrow/dataset/file_ipc.h +++ b/cpp/src/arrow/dataset/file_ipc.h @@ -90,7 +90,8 @@ class ARROW_DS_EXPORT IpcFileWriteOptions : public FileWriteOptions { std::shared_ptr metadata; protected: - using FileWriteOptions::FileWriteOptions; + explicit IpcFileWriteOptions(std::shared_ptr format) + : FileWriteOptions(std::move(format)) {} friend class IpcFileFormat; }; diff --git a/cpp/src/arrow/dataset/file_parquet.h b/cpp/src/arrow/dataset/file_parquet.h index 6f2f54206812e..6167c2894f65a 100644 --- a/cpp/src/arrow/dataset/file_parquet.h +++ b/cpp/src/arrow/dataset/file_parquet.h @@ -226,7 +226,8 @@ class ARROW_DS_EXPORT ParquetFileWriteOptions : public FileWriteOptions { std::shared_ptr arrow_writer_properties; protected: - using FileWriteOptions::FileWriteOptions; + explicit ParquetFileWriteOptions(std::shared_ptr format) + : FileWriteOptions(std::move(format)) {} friend class ParquetFileFormat; }; diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc index f8682ad313800..dff6fd830762f 100644 --- a/cpp/src/arrow/memory_pool.cc +++ b/cpp/src/arrow/memory_pool.cc @@ -229,6 +229,8 @@ class DebugAllocator { *out = memory_pool::internal::kZeroSizeArea; } else { ARROW_ASSIGN_OR_RAISE(int64_t raw_size, RawSize(size)); + DCHECK(raw_size > size) << "bug in raw size computation: " << raw_size + << " for size " << size; RETURN_NOT_OK(WrappedAllocator::AllocateAligned(raw_size, out)); InitAllocatedArea(*out, size); } @@ -250,6 +252,8 @@ class DebugAllocator { return Status::OK(); } ARROW_ASSIGN_OR_RAISE(int64_t raw_new_size, RawSize(new_size)); + DCHECK(raw_new_size > new_size) + << "bug in raw size computation: " << raw_new_size << " for size " << new_size; RETURN_NOT_OK( WrappedAllocator::ReallocateAligned(old_size + kOverhead, raw_new_size, ptr)); InitAllocatedArea(*ptr, new_size); diff --git a/cpp/src/arrow/util/aligned_storage.h b/cpp/src/arrow/util/aligned_storage.h index 213d8c72e4869..d398ee39df7a2 100644 --- a/cpp/src/arrow/util/aligned_storage.h +++ b/cpp/src/arrow/util/aligned_storage.h @@ -123,7 +123,26 @@ class AlignedStorage { } private: +#if !defined(__clang__) && defined(__GNUC__) && defined(__i386__) + // Workaround for GCC bug on i386: + // alignof(int64 | float64) can give different results depending on the + // compilation context, leading to internal ABI mismatch manifesting + // in incorrect propagation of Result between + // compilation units. + // (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88115) + static constexpr size_t alignment() { + if (std::is_integral_v && sizeof(T) == 8) { + return 4; + } else if (std::is_floating_point_v && sizeof(T) == 8) { + return 4; + } + return alignof(T); + } + + typename std::aligned_storage::type data_; +#else typename std::aligned_storage::type data_; +#endif }; } // namespace internal diff --git a/cpp/src/arrow/util/int_util_test.cc b/cpp/src/arrow/util/int_util_test.cc index 1a5fa472619b2..7217c1097e48e 100644 --- a/cpp/src/arrow/util/int_util_test.cc +++ b/cpp/src/arrow/util/int_util_test.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -594,5 +595,59 @@ TEST(CheckIntegersInRange, UnsignedInts) { CheckInRangeFails(uint64(), "[0, 10000000000, 10000000000]", "[0, 9999999999]"); } +template +class TestAddWithOverflow : public ::testing::Test { + public: + void CheckOk(T a, T b, T expected_result = {}) { + ARROW_SCOPED_TRACE("a = ", a, ", b = ", b); + T result; + ASSERT_FALSE(AddWithOverflow(a, b, &result)); + ASSERT_EQ(result, expected_result); + } + + void CheckOverflow(T a, T b) { + ARROW_SCOPED_TRACE("a = ", a, ", b = ", b); + T result; + ASSERT_TRUE(AddWithOverflow(a, b, &result)); + } +}; + +using SignedIntegerTypes = ::testing::Types; + +TYPED_TEST_SUITE(TestAddWithOverflow, SignedIntegerTypes); + +TYPED_TEST(TestAddWithOverflow, Basics) { + using T = TypeParam; + + const T almost_max = std::numeric_limits::max() - T{2}; + const T almost_min = std::numeric_limits::min() + T{2}; + + this->CheckOk(T{1}, T{2}, T{3}); + this->CheckOk(T{-1}, T{2}, T{1}); + this->CheckOk(T{-1}, T{-2}, T{-3}); + + this->CheckOk(almost_min, T{0}, almost_min); + this->CheckOk(almost_min, T{-2}, almost_min - T{2}); + this->CheckOk(almost_min, T{1}, almost_min + T{1}); + this->CheckOverflow(almost_min, T{-3}); + this->CheckOverflow(almost_min, almost_min); + + this->CheckOk(almost_max, T{0}, almost_max); + this->CheckOk(almost_max, T{2}, almost_max + T{2}); + this->CheckOk(almost_max, T{-1}, almost_max - T{1}); + this->CheckOverflow(almost_max, T{3}); + this->CheckOverflow(almost_max, almost_max); + + // In 2's complement, almost_min == - almost_max - 1 + this->CheckOk(almost_min, almost_max, T{-1}); + this->CheckOk(almost_max, almost_min, T{-1}); + this->CheckOk(almost_min - T{1}, almost_max, T{-2}); + this->CheckOk(almost_min + T{1}, almost_max, T{0}); + this->CheckOk(almost_min + T{2}, almost_max, T{1}); + this->CheckOk(almost_min, almost_max - T{1}, T{-2}); + this->CheckOk(almost_min, almost_max + T{1}, T{0}); + this->CheckOk(almost_min, almost_max + T{2}, T{1}); +} + } // namespace internal } // namespace arrow diff --git a/cpp/src/arrow/util/launder.h b/cpp/src/arrow/util/launder.h index 37e2a71443ad6..9e4533c4b4760 100644 --- a/cpp/src/arrow/util/launder.h +++ b/cpp/src/arrow/util/launder.h @@ -22,7 +22,7 @@ namespace arrow { namespace internal { -#if __cplusplus >= 201703L +#if __cpp_lib_launder using std::launder; #else template diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index 3794b7a6c435c..9d548142a7cee 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -27,17 +27,6 @@ add_dependencies(gandiva-all gandiva gandiva-tests gandiva-benchmarks) find_package(LLVMAlt REQUIRED) -if(LLVM_VERSION_MAJOR LESS "10") - set(GANDIVA_CXX_STANDARD ${CMAKE_CXX_STANDARD}) -else() - # LLVM 10 or later requires C++ 14 - if(CMAKE_CXX_STANDARD LESS 14) - set(GANDIVA_CXX_STANDARD 14) - else() - set(GANDIVA_CXX_STANDARD ${CMAKE_CXX_STANDARD}) - endif() -endif() - add_definitions(-DGANDIVA_LLVM_VERSION=${LLVM_VERSION_MAJOR}) find_package(OpenSSLAlt REQUIRED) @@ -172,7 +161,6 @@ add_arrow_lib(gandiva foreach(LIB_TARGET ${GANDIVA_LIBRARIES}) target_compile_definitions(${LIB_TARGET} PRIVATE GANDIVA_EXPORTING) - set_target_properties(${LIB_TARGET} PROPERTIES CXX_STANDARD ${GANDIVA_CXX_STANDARD}) endforeach() if(ARROW_BUILD_STATIC AND WIN32) @@ -234,7 +222,6 @@ function(ADD_GANDIVA_TEST REL_TEST_NAME) set(TEST_NAME gandiva-${REL_TEST_NAME}) string(REPLACE "_" "-" TEST_NAME ${TEST_NAME}) - set_target_properties(${TEST_NAME} PROPERTIES CXX_STANDARD ${GANDIVA_CXX_STANDARD}) endfunction() add_gandiva_test(internals-test diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt index fefabf883657a..d7c7ef157b442 100644 --- a/cpp/src/gandiva/precompiled/CMakeLists.txt +++ b/cpp/src/gandiva/precompiled/CMakeLists.txt @@ -43,10 +43,10 @@ if(MSVC) else() message(FATAL_ERROR "Unsupported MSVC_VERSION=${MSVC_VERSION}") endif() - set(PLATFORM_CLANG_OPTIONS -std=c++14 -fms-compatibility + set(PLATFORM_CLANG_OPTIONS -std=c++17 -fms-compatibility -fms-compatibility-version=${FMS_COMPATIBILITY}) else() - set(PLATFORM_CLANG_OPTIONS -std=c++11) + set(PLATFORM_CLANG_OPTIONS -std=c++17) endif() # Create bitcode for each of the source files. diff --git a/cpp/src/gandiva/symbols.map b/cpp/src/gandiva/symbols.map index 77f0001068200..dbce381f3e295 100644 --- a/cpp/src/gandiva/symbols.map +++ b/cpp/src/gandiva/symbols.map @@ -25,9 +25,9 @@ extern "C++" { # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically - # links c++11 symbols into binaries so that the result may be executed on + # links c++1x symbols into binaries so that the result may be executed on # a system with an older libstdc++ which doesn't include the necessary - # c++11 symbols. + # c++1x symbols. std::*; *std::__once_call*; }; diff --git a/cpp/src/parquet/symbols.map b/cpp/src/parquet/symbols.map index 4bf032dd5846c..a91686ef0ecda 100644 --- a/cpp/src/parquet/symbols.map +++ b/cpp/src/parquet/symbols.map @@ -31,9 +31,9 @@ apache::thrift::*; # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically - # links c++11 symbols into binaries so that the result may be executed on + # links c++1x symbols into binaries so that the result may be executed on # a system with an older libstdc++ which doesn't include the necessary - # c++11 symbols. + # c++1x symbols. std::*; *std::__once_call*; }; diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index 5784b78a8a05c..8045254871cec 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -209,7 +209,8 @@ def _git_ssh_to_https(url): def _parse_github_user_repo(remote_url): - m = re.match(r'.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', remote_url) + # TODO: use a proper URL parser instead? + m = re.match(r'.*\/([^\/]+)\/([^\/\.]+)(\.git|/)?$', remote_url) if m is None: # Perhaps it's simply "username/reponame"? m = re.match(r'^(\w+)/(\w+)$', remote_url) diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh index 5de5209a55ce5..69229b012a886 100755 --- a/dev/release/verify-apt.sh +++ b/dev/release/verify-apt.sh @@ -151,7 +151,7 @@ pushd build/minimal_build cmake . make -j$(nproc) ./arrow-example -c++ -std=c++11 -o arrow-example example.cc $(pkg-config --cflags --libs arrow) +c++ -std=c++17 -o arrow-example example.cc $(pkg-config --cflags --libs arrow) ./arrow-example popd echo "::endgroup::" diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh index 6d40bccf2bc57..118a0e13ad7cd 100755 --- a/dev/release/verify-yum.sh +++ b/dev/release/verify-yum.sh @@ -46,6 +46,8 @@ repository_version="${distribution_version}" cmake_package=cmake cmake_command=cmake +devtoolset= +scl_package= have_arrow_libs=no have_flight=yes have_gandiva=yes @@ -93,6 +95,8 @@ case "${distribution}-${distribution_version}" in distribution_prefix="centos" cmake_package=cmake3 cmake_command=cmake3 + devtoolset=11 + scl_package=centos-release-scl-rh have_arrow_libs=yes have_flight=no have_gandiva=no @@ -192,20 +196,31 @@ echo "::endgroup::" echo "::group::Test Apache Arrow C++" ${install_command} --enablerepo=epel arrow-devel-${package_version} +if [ -n "${devtoolset}" ]; then + ${install_command} ${scl_package} +fi ${install_command} \ ${cmake_package} \ - gcc-c++ \ git \ libarchive \ - make \ pkg-config +if [ -n "${devtoolset}" ]; then + ${install_command} \ + devtoolset-${devtoolset}-gcc-c++ \ + devtoolset-${devtoolset}-make + . /opt/rh/devtoolset-${devtoolset}/enable +else + ${install_command} \ + gcc-c++ \ + make +fi mkdir -p build cp -a /arrow/cpp/examples/minimal_build build/ pushd build/minimal_build ${cmake_command} . make -j$(nproc) ./arrow-example -c++ -std=c++11 -o arrow-example example.cc $(pkg-config --cflags --libs arrow) +c++ -std=c++17 -o arrow-example example.cc $(pkg-config --cflags --libs arrow) ./arrow-example popd echo "::endgroup::" diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.18python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.18python3.7.____cpython.yaml index bcfe18d267182..2592e75f0c02a 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.18python3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.18python3.7.____cpython.yaml @@ -1,7 +1,7 @@ bzip2: - '1' c_compiler: -- vs2017 +- vs2019 channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ cuda_compiler: cuda_compiler_version: - '10.2' cxx_compiler: -- vs2017 +- vs2019 gflags: - '2.2' glog: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.18python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.18python3.8.____cpython.yaml index 766615b6e4fcc..3aa9cca60aac4 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.18python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.18python3.8.____cpython.yaml @@ -1,7 +1,7 @@ bzip2: - '1' c_compiler: -- vs2017 +- vs2019 channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ cuda_compiler: cuda_compiler_version: - '10.2' cxx_compiler: -- vs2017 +- vs2019 gflags: - '2.2' glog: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml index 2001de807943d..b01a67017fa99 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml @@ -1,7 +1,7 @@ bzip2: - '1' c_compiler: -- vs2017 +- vs2019 channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ cuda_compiler: cuda_compiler_version: - '10.2' cxx_compiler: -- vs2017 +- vs2019 gflags: - '2.2' glog: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.21python3.10.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.21python3.10.____cpython.yaml index a7c9c54c2e8e5..fb868730af27a 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.21python3.10.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.21python3.10.____cpython.yaml @@ -1,7 +1,7 @@ bzip2: - '1' c_compiler: -- vs2017 +- vs2019 channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ cuda_compiler: cuda_compiler_version: - '10.2' cxx_compiler: -- vs2017 +- vs2019 gflags: - '2.2' glog: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.18python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.18python3.7.____cpython.yaml index a23b37e3ae2f0..b9fb499fe5eca 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.18python3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.18python3.7.____cpython.yaml @@ -1,7 +1,7 @@ bzip2: - '1' c_compiler: -- vs2017 +- vs2019 channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ cuda_compiler: cuda_compiler_version: - None cxx_compiler: -- vs2017 +- vs2019 gflags: - '2.2' glog: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.18python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.18python3.8.____cpython.yaml index dbf6186a4d256..806d2363decc3 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.18python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.18python3.8.____cpython.yaml @@ -1,7 +1,7 @@ bzip2: - '1' c_compiler: -- vs2017 +- vs2019 channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ cuda_compiler: cuda_compiler_version: - None cxx_compiler: -- vs2017 +- vs2019 gflags: - '2.2' glog: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml index 7341d646b6b46..990696de9ff4a 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml @@ -1,7 +1,7 @@ bzip2: - '1' c_compiler: -- vs2017 +- vs2019 channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ cuda_compiler: cuda_compiler_version: - None cxx_compiler: -- vs2017 +- vs2019 gflags: - '2.2' glog: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython.yaml index 374e7ce82a70a..cc9069c0bed50 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython.yaml @@ -1,7 +1,7 @@ bzip2: - '1' c_compiler: -- vs2017 +- vs2019 channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ cuda_compiler: cuda_compiler_version: - None cxx_compiler: -- vs2017 +- vs2019 gflags: - '2.2' glog: diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat index 7800123e7a0f6..5ef5d01acc672 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat @@ -37,9 +37,9 @@ cmake -G "Ninja" ^ -DARROW_MIMALLOC:BOOL=ON ^ -DARROW_DATASET:BOOL=ON ^ -DARROW_FLIGHT:BOOL=ON ^ - -DARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS:BOOL=ON ^ + -DARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS:BOOL=OFF ^ -DARROW_HDFS:BOOL=ON ^ - -DARROW_GCS:BOOL=ON ^ + -DARROW_GCS:BOOL=OFF ^ -DARROW_PARQUET:BOOL=ON ^ -DPARQUET_REQUIRE_ENCRYPTION:BOOL=ON ^ -DARROW_GANDIVA:BOOL=ON ^ diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat index a03a37722fae0..8897c4b7d4b79 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat @@ -17,7 +17,7 @@ pushd "%SRC_DIR%"\python SET ARROW_HOME=%LIBRARY_PREFIX% SET SETUPTOOLS_SCM_PRETEND_VERSION=%PKG_VERSION% SET PYARROW_BUILD_TYPE=release -SET PYARROW_WITH_GCS=1 +SET PYARROW_WITH_GCS=0 SET PYARROW_WITH_S3=1 SET PYARROW_WITH_HDFS=1 SET PYARROW_WITH_DATASET=1 diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index 791a82c4ba9b5..7b13e92235c02 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -7,11 +7,14 @@ mkdir cpp/build pushd cpp/build EXTRA_CMAKE_ARGS="" +ARROW_GCS="OFF" -# Include g++'s system headers if [ "$(uname)" == "Linux" ]; then + # Include g++'s system headers SYSTEM_INCLUDES=$(echo | ${CXX} -E -Wp,-v -xc++ - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') EXTRA_CMAKE_ARGS=" -DARROW_GANDIVA_PC_CXX_FLAGS=${SYSTEM_INCLUDES}" + # GCS doesn't produce any abseil-induced linker error on Linux, enable it + ARROW_GCS="ON" fi # Enable CUDA support @@ -47,11 +50,9 @@ else EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=ON" fi -if [[ "${target_platform}" == osx-* ]]; then - EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_CXX_STANDARD=14" -else - EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_CXX_STANDARD=17" -fi +# See https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk +CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" +ARROW_GANDIVA_PC_CXX_FLAGS="-D_LIBCPP_DISABLE_AVAILABILITY" cmake \ -DARROW_BOOST_USE_SHARED=ON \ @@ -60,11 +61,13 @@ cmake \ -DARROW_BUILD_TESTS=OFF \ -DARROW_BUILD_UTILITIES=OFF \ -DBUILD_SHARED_LIBS=ON \ + -DARROW_CXXFLAGS="${CXXFLAGS}" \ + -DARROW_GANDIVA_PC_CXX_FLAGS="${ARROW_GANDIVA_PC_CXX_FLAGS}" \ -DARROW_DATASET=ON \ -DARROW_DEPENDENCY_SOURCE=SYSTEM \ -DARROW_FLIGHT=ON \ -DARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS=ON \ - -DARROW_GCS=ON \ + -DARROW_GCS=${ARROW_GCS} \ -DARROW_HDFS=ON \ -DARROW_JEMALLOC=ON \ -DARROW_MIMALLOC=ON \ @@ -84,6 +87,7 @@ cmake \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ -DCMAKE_BUILD_TYPE=release \ + -DCMAKE_CXX_STANDARD=17 \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ -DLLVM_TOOLS_BINARY_DIR=$PREFIX/bin \ diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh index 6e23c5eed904b..fc28bb0623e57 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh @@ -17,7 +17,7 @@ if [[ "${target_platform}" == "osx-arm64" ]]; then else export PYARROW_WITH_GANDIVA=1 fi -export PYARROW_WITH_GCS=1 +export PYARROW_WITH_GCS=0 export PYARROW_WITH_HDFS=1 export PYARROW_WITH_ORC=1 export PYARROW_WITH_PARQUET=1 @@ -28,6 +28,10 @@ export PYARROW_CMAKE_GENERATOR=Ninja export PYARROW_CMAKE_OPTIONS="-DARROW_SIMD_LEVEL=NONE" BUILD_EXT_FLAGS="" +if [ "$(uname)" == "Linux" ]; then + export PYARROW_WITH_GCS=1 +fi + # Enable CUDA support if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]]; then export PYARROW_WITH_CUDA=1 @@ -40,6 +44,9 @@ if [[ "${target_platform}" == "linux-aarch64" ]]; then export PYARROW_CMAKE_OPTIONS="-DARROW_ARMV8_ARCH=armv8-a ${PYARROW_CMAKE_OPTIONS}" fi +# See https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk +export CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" + cd python $PYTHON setup.py \ diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 59a3de085d972..87eb034990660 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -75,7 +75,10 @@ outputs: - c-ares - gflags - glog - - google-cloud-cpp + # On macOS and Windows, GCS support fails linking due to ABI + # issues with abseil-cpp + # (see https://github.com/conda-forge/abseil-cpp-feedstock/issues/45) + - google-cloud-cpp # [linux] - grpc-cpp - libprotobuf - clangdev 10 # [not (osx and arm64)] @@ -217,7 +220,7 @@ outputs: - pyarrow.parquet - pyarrow.plasma # [unix] - pyarrow.fs - - pyarrow._s3fs + - pyarrow._s3fs # [linux] - pyarrow._hdfs # We can only test importing cuda package but cannot run when a # CUDA device is not available, for instance, when building from CI. diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index c8c534c47322f..6aac0eeda9066 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -13,28 +13,6 @@ jobs: UPLOAD_PACKAGES: False steps: - - script: | - choco install vcpython27 -fdv -y --debug - condition: contains(variables['CONFIG'], 'vs2008') - displayName: Install vcpython27.msi (if needed) - - - powershell: | - Set-PSDebug -Trace 1 - $batchcontent = @" - ECHO ON - SET vcpython=C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0 - DIR "%vcpython%" - CALL "%vcpython%\vcvarsall.bat" %* - "@ - $batchDir = "C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC" - $batchPath = "$batchDir" + "\vcvarsall.bat" - New-Item -Path $batchPath -ItemType "file" -Force - Set-Content -Value $batchcontent -Path $batchPath - Get-ChildItem -Path $batchDir - Get-ChildItem -Path ($batchDir + '\..') - condition: contains(variables['CONFIG'], 'vs2008') - displayName: Patch vs2008 (if needed) - - task: PythonScript@0 displayName: 'Download Miniforge' inputs: @@ -48,6 +26,7 @@ jobs: - script: | start /wait "" %BUILD_ARTIFACTSTAGINGDIRECTORY%\Miniforge.exe /InstallationType=JustMe /RegisterPython=0 /S /D=C:\Miniforge displayName: Install Miniforge + - powershell: Write-Host "##vso[task.prependpath]C:\Miniforge\Scripts" displayName: Add conda to PATH @@ -83,7 +62,6 @@ jobs: workingDirectory: arrow\dev\tasks\conda-recipes env: PYTHONUNBUFFERED: 1 - condition: not(contains(variables['CONFIG'], 'vs2008')) - script: | call activate base diff --git a/dev/tasks/conda-recipes/r-arrow/build.sh b/dev/tasks/conda-recipes/r-arrow/build.sh index e868189a21fd2..724a8afd4fb2e 100644 --- a/dev/tasks/conda-recipes/r-arrow/build.sh +++ b/dev/tasks/conda-recipes/r-arrow/build.sh @@ -1,3 +1,9 @@ #!/usr/bin/env bash + +set -ex + export DISABLE_AUTOBREW=1 +# See https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk +export ARROW_R_CXXFLAGS="${ARROW_R_CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" + $R CMD INSTALL --build r/. diff --git a/dev/tasks/conda-recipes/r-arrow/configure.win b/dev/tasks/conda-recipes/r-arrow/configure.win index 173a129b3bf99..fb16a810b8f22 100755 --- a/dev/tasks/conda-recipes/r-arrow/configure.win +++ b/dev/tasks/conda-recipes/r-arrow/configure.win @@ -4,5 +4,5 @@ set -euxo pipefail echo "PKG_CPPFLAGS=-DNDEBUG -I\"${LIBRARY_PREFIX}/include\" -I\"${PREFIX}/include\" -DARROW_R_WITH_PARQUET -DARROW_R_WITH_DATASET -DARROW_R_WITH_S3 -DARROW_R_WITH_JSON" > src/Makevars.win echo "PKG_CXXFLAGS=\$(CXX_VISIBILITY)" >> src/Makevars.win -echo 'CXX_STD=CXX11' >> src/Makevars.win +echo 'CXX_STD=CXX17' >> src/Makevars.win echo "PKG_LIBS=-L\"${LIBRARY_PREFIX}/lib\" -larrow_dataset -lparquet -larrow" >> src/Makevars.win diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb index e68f939bcab4c..86b08ff6fbe2e 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow.rb @@ -111,7 +111,7 @@ def install return 0; } EOS - system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", "-larrow", "-o", "test" + system ENV.cxx, "test.cpp", "-std=c++17", "-I#{include}", "-L#{lib}", "-larrow", "-o", "test" system "./test" end end diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb index 79d1da2c212d4..84206978f3569 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb @@ -91,7 +91,7 @@ def install return 0; } EOS - system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", \ + system ENV.cxx, "test.cpp", "-std=c++17", "-I#{include}", "-L#{lib}", \ "-larrow", "-larrow_bundled_dependencies", "-o", "test" system "./test" end diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index c5e2ca557c96d..1530a7b1d9713 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -82,7 +82,7 @@ def install return 0; } EOS - system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", \ + system ENV.cxx, "test.cpp", "-std=c++17", "-I#{include}", "-L#{lib}", \ "-larrow", "-larrow_bundled_dependencies", "-o", "test" system "./test" end diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 3d72d23b90381..705ef73a0addf 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -22,6 +22,8 @@ %define _amzn %{?amzn:%{amzn}}%{!?amzn:0} %define is_amazon_linux (%{_amzn} != 0) +%define is_centos_7 (%{rhel} == 7 && !%{is_amazon_linux}) + %define major_version %(echo @VERSION@ | grep -o '^[0-9]*') %define boost_version %( \ @@ -56,6 +58,12 @@ %define arrow_cmake_install DESTDIR="%{buildroot}" make -C %{arrow_cmake_builddir} install %endif +%if %{is_centos_7} +%define gcc_package devtoolset-11-gcc +%else +%define gcc_package gcc +%endif + %define use_boost (!%{is_amazon_linux}) # TODO: Enable this on aarch64 too. This causes timeout on Travis CI. %define use_flight (%{rhel} >= 8 && "%{_arch}" != "aarch64") @@ -96,7 +104,7 @@ BuildRequires: c-ares-devel BuildRequires: cmake%{cmake_version} BuildRequires: curl-devel BuildRequires: flex -BuildRequires: gcc-c++ +BuildRequires: %{gcc_package}-c++ %if %{use_gflags} BuildRequires: gflags-devel %endif diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile index 04e74012f9521..1da8e0fb79621 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -18,13 +18,20 @@ ARG FROM=centos:7 FROM ${FROM} +ENV \ + SCL=devtoolset-11 + ARG DEBUG RUN \ quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ yum update -y ${quiet} && \ - yum install -y ${quiet} epel-release && \ yum install -y ${quiet} \ + centos-release-scl-rh \ + epel-release && \ + yum install -y ${quiet} \ + ${SCL}-gcc-c++ \ + ${SCL}-make \ bison \ boost169-devel \ brotli-devel \ @@ -33,7 +40,6 @@ RUN \ cmake3 \ curl-devel \ flex \ - gcc-c++ \ gflags-devel \ git \ glog-devel \ @@ -42,7 +48,6 @@ RUN \ json-devel \ libzstd-devel \ lz4-devel \ - make \ ninja-build \ openssl-devel \ pkg-config \ diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 4cf4830cf650e..af3f6105645fc 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -314,7 +314,8 @@ on: path: repo/libarrow/bin/windows {% endif %} {% if get_nix %} - {% for os in ["ubuntu-18.04", "ubuntu-22.04", "centos-7"] %} + # TODO(ARROW-17594): re-enable centos-7 here + {% for os in ["ubuntu-18.04", "ubuntu-22.04"] %} - name: Get {{ os }} binary uses: actions/download-artifact@v3 with: diff --git a/dev/tasks/r/azure.linux.yml b/dev/tasks/r/azure.linux.yml index fd481419610f0..4528390298540 100644 --- a/dev/tasks/r/azure.linux.yml +++ b/dev/tasks/r/azure.linux.yml @@ -42,7 +42,7 @@ jobs: export R_ORG={{ r_org }} export R_IMAGE={{ r_image }} export R_TAG={{ r_tag }} - export DEVTOOLSET_VERSION={{ devtoolset_version|default("-1") }} + export DEVTOOLSET_VERSION={{ devtoolset_version|default("") }} export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }} docker-compose pull --ignore-pull-failures r docker-compose build r diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 090735f0fa11a..30dd0641d21e1 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -70,7 +70,8 @@ jobs: config: - { os: ubuntu, version: "18.04" } - { os: ubuntu, version: "22.04" } - - { os: centos, version: "7" } + # TODO(ARROW-17594): re-enable this with the devtoolset for C++17 + #- { os: centos, version: "7" } env: UBUNTU: {{ '${{ matrix.config.version }}' }} steps: @@ -240,7 +241,8 @@ jobs: - {image: "rocker/r-ver:4.0.0"} # ubuntu-20.04 - {image: "rocker/r-ver:3.6.3", libarrow_binary: "TRUE"} # debian:buster (10) - {image: "rstudio/r-base:4.1-bionic"} # ubuntu-18.04 - - {image: "rstudio/r-base:4.2-centos7"} + # TODO(ARROW-17594): re-enable this with the devtoolset for C++17 + #- {image: "rstudio/r-base:4.2-centos7"} # Even with devtoolset-8, binaries fail because glibc is older than # what we build with on ubuntu-18.04 (uncomment to see the error) # TODO(ARROW-16976): build binaries with older glibc diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index ae3c613902bac..a0bb7e304b713 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -87,7 +87,7 @@ groups: # r-conda tasks - conda-linux-gcc-py*-cpu-r* - conda-osx-clang-py*-r* - - conda-win-vs2017-py*-r* + - conda-win-vs2019-py*-r* - r-binary-packages ruby: @@ -408,7 +408,7 @@ tasks: ############################## Conda Windows ################################ - conda-win-vs2017-py37-r40: + conda-win-vs2019-py37-r40: ci: azure template: conda-recipes/azure.win.yml params: @@ -418,7 +418,7 @@ tasks: - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - conda-win-vs2017-py37-r41: + conda-win-vs2019-py37-r41: ci: azure template: conda-recipes/azure.win.yml params: @@ -428,7 +428,7 @@ tasks: - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - conda-win-vs2017-py38: + conda-win-vs2019-py38: ci: azure template: conda-recipes/azure.win.yml params: @@ -437,7 +437,7 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - conda-win-vs2017-py39: + conda-win-vs2019-py39: ci: azure template: conda-recipes/azure.win.yml params: @@ -446,7 +446,7 @@ tasks: - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - conda-win-vs2017-py310: + conda-win-vs2019-py310: ci: azure template: conda-recipes/azure.win.yml params: @@ -1202,7 +1202,7 @@ tasks: flags: "-e ARROW_BUILD_SHARED=OFF -e ARROW_BUILD_STATIC=ON -e ARROW_TEST_LINKAGE=static" image: ubuntu-cpp -{% for cpp_standard in [14, 17] %} +{% for cpp_standard in [17] %} test-ubuntu-20.04-cpp-{{ cpp_standard }}: ci: github template: docker-tests/github.linux.yml diff --git a/dev/tasks/verify-rc/github.macos.amd64.yml b/dev/tasks/verify-rc/github.macos.amd64.yml index 3ab9be3157657..cdd6170289915 100644 --- a/dev/tasks/verify-rc/github.macos.amd64.yml +++ b/dev/tasks/verify-rc/github.macos.amd64.yml @@ -21,6 +21,10 @@ {% set use_conda = use_conda|default(False) %} +env: + # Current oldest supported version according to https://endoflife.date/macos + MACOSX_DEPLOYMENT_TARGET: "10.15" + jobs: verify: name: "Verify release candidate on macOS" diff --git a/docs/source/cpp/conventions.rst b/docs/source/cpp/conventions.rst index 07af8f0937ab6..ba61d747db8ab 100644 --- a/docs/source/cpp/conventions.rst +++ b/docs/source/cpp/conventions.rst @@ -29,8 +29,7 @@ there may be exceptions. Language version ---------------- -Arrow is C++11-compatible. A few backports are used for newer functionality, -for example the :class:`std::string_view` class. +Starting with version 10.0, Arrow C++ is C++17-compatible. Namespacing ----------- diff --git a/docs/source/developers/cpp/building.rst b/docs/source/developers/cpp/building.rst index b988bd2eebe11..0ef61685ced81 100644 --- a/docs/source/developers/cpp/building.rst +++ b/docs/source/developers/cpp/building.rst @@ -39,8 +39,8 @@ out-of-source. If you are not familiar with this terminology: Building requires: -* A C++11-enabled compiler. On Linux, gcc 4.8 and higher should be - sufficient. For Windows, at least Visual Studio 2017 is required. +* A C++17-enabled compiler. On Linux, gcc 7.1 and higher should be + sufficient. For Windows, at least Visual Studio VS2017 is required. * CMake 3.5 or higher * On Linux and macOS, either ``make`` or ``ninja`` build utilities * At least 1GB of RAM for a minimal build, 4GB for a minimal diff --git a/docs/source/java/cdata.rst b/docs/source/java/cdata.rst index 44e4f230ba65d..94c86d6a4f821 100644 --- a/docs/source/java/cdata.rst +++ b/docs/source/java/cdata.rst @@ -154,7 +154,7 @@ without writing JNI bindings ourselves. include = { "CDataCppBridge.h" }, - compiler = {"cpp11"}, + compiler = {"cpp17"}, linkpath = {"/arrow/cpp/build/debug/"}, link = {"arrow"} ) @@ -436,7 +436,7 @@ CMakeLists.txt definition file: find_package(Arrow REQUIRED) message(STATUS "Arrow version: ${ARROW_VERSION}") include_directories(${JNI_INCLUDE_DIRS}) - set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD 17) add_executable(${PROJECT_NAME} main.cpp) target_link_libraries(cdatacpptojava PRIVATE arrow_shared) target_link_libraries(cdatacpptojava PRIVATE ${JNI_LIBRARIES}) diff --git a/docs/source/python/integration/extending.rst b/docs/source/python/integration/extending.rst index ec945e97ce2c0..8931960f3a9eb 100644 --- a/docs/source/python/integration/extending.rst +++ b/docs/source/python/integration/extending.rst @@ -434,12 +434,7 @@ To build this module, you will need a slightly customized ``setup.py`` file ext.library_dirs.extend(pa.get_library_dirs()) if os.name == 'posix': - ext.extra_compile_args.append('-std=c++11') - - # Try uncommenting the following line on Linux - # if you get weird linker errors or runtime crashes - # ext.define_macros.append(("_GLIBCXX_USE_CXX11_ABI", "0")) - + ext.extra_compile_args.append('-std=c++17') setup(ext_modules=ext_modules) @@ -476,7 +471,3 @@ the CentOS `devtoolset-9`. In addition to the other notes above, if you are compiling C++ using these shared libraries, you will need to make sure you use a compatible toolchain as well or you might see a segfault during runtime. - -Also, if you encounter errors when linking or loading the library, consider -setting the ``_GLIBCXX_USE_CXX11_ABI`` preprocessor macro to ``0`` -(for example by adding ``-D_GLIBCXX_USE_CXX11_ABI=0`` to ``CFLAGS``). diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index f187cd943d16a..4778f030c2588 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -21,7 +21,7 @@ message(STATUS "Building using CMake version: ${CMAKE_VERSION}") project(arrow-java-jni) if("${CMAKE_CXX_STANDARD}" STREQUAL "") - set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD 17) endif() set(CMAKE_CXX_STANDARD_REQUIRED ON) diff --git a/java/c/README.md b/java/c/README.md index ce73f531c1b31..db2a2403b374f 100644 --- a/java/c/README.md +++ b/java/c/README.md @@ -24,7 +24,7 @@ install: - Java 8 or later - Maven 3.3 or later - - A C++11-enabled compiler + - A C++17-enabled compiler - CMake 3.11 or later - Make or ninja build utilities diff --git a/matlab/CMakeLists.txt b/matlab/CMakeLists.txt index 79bc809a21fce..1b9783e89487c 100644 --- a/matlab/CMakeLists.txt +++ b/matlab/CMakeLists.txt @@ -187,7 +187,7 @@ macro(build_gtest) add_dependencies(GTest::gtest_main arrow_ep) endmacro() -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) set(MLARROW_VERSION "10.0.0-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" MLARROW_BASE_VERSION "${MLARROW_VERSION}") diff --git a/matlab/README.md b/matlab/README.md index 6f99fc4f5f9d6..3c183f7b1644e 100644 --- a/matlab/README.md +++ b/matlab/README.md @@ -31,7 +31,7 @@ To build the MATLAB Interface to Apache Arrow from source, the following softwar 1. [MATLAB](https://www.mathworks.com/products/get-matlab.html) 2. [CMake](https://cmake.org/cmake/help/latest/) -3. C++ compiler which supports C++11 (e.g. [`gcc`](https://gcc.gnu.org/) on Linux, [`Xcode`](https://developer.apple.com/xcode/) on macOS, or [`Visual Studio`](https://visualstudio.microsoft.com/) on Windows) +3. C++ compiler which supports C++17 (e.g. [`gcc`](https://gcc.gnu.org/) on Linux, [`Xcode`](https://developer.apple.com/xcode/) on macOS, or [`Visual Studio`](https://visualstudio.microsoft.com/) on Windows) 4. [Git](https://git-scm.com/) ## Setup diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 2951e3de8f83e..e2c53974cfe68 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -170,9 +170,12 @@ endif() # For any C code, use the same flags. set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}") -# Add C++-only flags, like -std=c++11 +# Add C++-only flags, like -std=c++17 set(CMAKE_CXX_FLAGS "${CXX_ONLY_FLAGS} ${CMAKE_CXX_FLAGS}") +message(STATUS "CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") +message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}") + if(MSVC) # MSVC makes its own output directories based on the build configuration set(BUILD_SUBDIR_NAME "") @@ -213,11 +216,6 @@ set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") # where to put generated binaries set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}") -if(PYARROW_USE_TENSORFLOW) - # TensorFlow uses the old GLIBCXX ABI, so we have to use it too - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") -endif() - # Python and Numpy libraries find_package(Python3Alt REQUIRED) include(UseCython) diff --git a/python/pyarrow/plasma.py b/python/pyarrow/plasma.py index 052c69975d3ca..712ed9e5b2261 100644 --- a/python/pyarrow/plasma.py +++ b/python/pyarrow/plasma.py @@ -64,7 +64,7 @@ def build_plasma_tensorflow_op(): tf_cflags = tf.sysconfig.get_compile_flags() if sys.platform == 'darwin': tf_cflags = ["-undefined", "dynamic_lookup"] + tf_cflags - cmd = ["g++", "-std=c++11", "-g", "-shared", cc_path, + cmd = ["g++", "-std=c++17", "-g", "-shared", cc_path, "-o", so_path, "-DNDEBUG", "-I" + pa.get_include()] cmd += ["-L" + dir for dir in pa.get_library_dirs()] cmd += ["-lplasma", "-larrow_python", "-larrow", "-fPIC"] diff --git a/python/pyarrow/src/CMakeLists.txt b/python/pyarrow/src/CMakeLists.txt index a1dd330603d54..552a194237677 100644 --- a/python/pyarrow/src/CMakeLists.txt +++ b/python/pyarrow/src/CMakeLists.txt @@ -185,17 +185,20 @@ endif() include(GNUInstallDirs) -# This ensures that things like gnu++11 get passed correctly +# This ensures that things like -std=gnu++... get passed correctly if(NOT DEFINED CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD 17) endif() -# We require a C++11 compliant compiler +# We require a C++17 compliant compiler set(CMAKE_CXX_STANDARD_REQUIRED ON) # Needed compiler flags include(SetupCxxFlags) +message(STATUS "CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") +message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}") + set(ARROW_PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS}) # Inlude macros needed to find and use add_arrow_lib function diff --git a/python/pyarrow/tests/test_cython.py b/python/pyarrow/tests/test_cython.py index 85d65383d8ba7..6c1c47f2a24d8 100644 --- a/python/pyarrow/tests/test_cython.py +++ b/python/pyarrow/tests/test_cython.py @@ -29,9 +29,9 @@ here = os.path.dirname(os.path.abspath(__file__)) test_ld_path = os.environ.get('PYARROW_TEST_LD_PATH', '') if os.name == 'posix': - compiler_opts = ['-std=c++11'] + compiler_opts = ['-std=c++17'] elif os.name == 'nt': - compiler_opts = ['-D_ENABLE_EXTENDED_ALIGNED_STORAGE'] + compiler_opts = ['-D_ENABLE_EXTENDED_ALIGNED_STORAGE', '/std:c++17'] else: compiler_opts = [] diff --git a/python/setup.py b/python/setup.py index fc868ff7addaf..05fc9423b5ad1 100755 --- a/python/setup.py +++ b/python/setup.py @@ -278,6 +278,8 @@ def append_cmake_bool(value, varname): # build args build_tool_args = [] + if os.environ.get('PYARROW_BUILD_VERBOSE', '0') == '1': + cmake_options.append('-DCMAKE_VERBOSE_MAKEFILE=ON') if os.environ.get('PYARROW_PARALLEL'): build_tool_args.append('--') build_tool_args.append( diff --git a/r/DESCRIPTION b/r/DESCRIPTION index a728be37734f7..7ae6a8de29fea 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -26,7 +26,7 @@ URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/ BugReports: https://issues.apache.org/jira/projects/ARROW/issues Encoding: UTF-8 Language: en-US -SystemRequirements: C++11; for AWS S3 support on Linux, libcurl and openssl (optional) +SystemRequirements: C++17; for AWS S3 support on Linux, libcurl and openssl (optional) Biarch: true Imports: assertthat, diff --git a/r/configure b/r/configure index eab213b631f3e..a1c36add732fd 100755 --- a/r/configure +++ b/r/configure @@ -208,22 +208,17 @@ if grep raspbian /etc/os-release >/dev/null 2>&1; then PKG_LIBS="-latomic $PKG_LIBS" fi -# If libarrow uses the old GLIBCXX ABI, so we have to use it too -if [ "$ARROW_USE_OLD_CXXABI" ]; then - PKG_CFLAGS="$PKG_CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0" -fi - # Set any user-defined CXXFLAGS if [ "$ARROW_R_CXXFLAGS" ]; then PKG_CFLAGS="$PKG_CFLAGS $ARROW_R_CXXFLAGS" fi # Test that we can find libarrow -CXX11="`${R_HOME}/bin/R CMD config CXX11` -E" -CXX11FLAGS=`"${R_HOME}"/bin/R CMD config CXX11FLAGS` -CXX11STD=`"${R_HOME}"/bin/R CMD config CXX11STD` +CXX17="`${R_HOME}/bin/R CMD config CXX17` -E" +CXX17FLAGS=`"${R_HOME}"/bin/R CMD config CXX17FLAGS` +CXX17STD=`"${R_HOME}"/bin/R CMD config CXX17STD` CPPFLAGS=`"${R_HOME}"/bin/R CMD config CPPFLAGS` -TEST_CMD="${CXX11} ${CPPFLAGS} ${PKG_CFLAGS} ${CXX11FLAGS} ${CXX11STD} -xc++ -" +TEST_CMD="${CXX17} ${CPPFLAGS} ${PKG_CFLAGS} ${CXX17FLAGS} ${CXX17STD} -xc++ -" echo "#include $PKG_TEST_HEADER" | ${TEST_CMD} >/dev/null 2>&1 if [ $? -eq 0 ]; then diff --git a/r/src/Makevars.in b/r/src/Makevars.in index 7e20f9561ed7c..0516d29726378 100644 --- a/r/src/Makevars.in +++ b/r/src/Makevars.in @@ -25,5 +25,5 @@ PKG_CPPFLAGS=@cflags@ # https://bugs.llvm.org/show_bug.cgi?id=39191 # https://www.mail-archive.com/gcc-bugs@gcc.gnu.org/msg534862.html # PKG_CXXFLAGS=$(CXX_VISIBILITY) -CXX_STD=CXX11 +CXX_STD=CXX17 PKG_LIBS=@libs@ diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index b76af09ec53bf..0757a34a0cd2c 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -192,7 +192,7 @@ compile_test_program <- function(code) { # Note: if we wanted to check for openssl on macOS, we'd have to set the brew # path as a -I directory. But since we (currently) only run this code to # determine whether we can download a Linux binary, it's not relevant. - runner <- "`R CMD config CXX11` `R CMD config CPPFLAGS` `R CMD config CXX11FLAGS` `R CMD config CXX11STD` -E -xc++" + runner <- "`R CMD config CXX17` `R CMD config CPPFLAGS` `R CMD config CXX17FLAGS` `R CMD config CXX17STD` -E -xc++" suppressWarnings(system2("echo", sprintf('"%s" | %s -', code, runner), stdout = FALSE, stderr = TRUE)) } @@ -240,7 +240,7 @@ header_not_found <- function(header, errs) { any(grepl(regex, errs)) } -compiler_version_string <- function(compiler = R_CMD_config("CXX11")) { +compiler_version_string <- function(compiler = R_CMD_config("CXX17")) { system(paste(compiler, "--version"), intern = TRUE) } @@ -431,8 +431,8 @@ build_libarrow <- function(src_dir, dst_dir) { # is found, it will be used by the libarrow build, and this does # not affect how R compiles the arrow bindings. CC = sub("^.*ccache", "", R_CMD_config("CC")), - CXX = paste(sub("^.*ccache", "", R_CMD_config("CXX11")), R_CMD_config("CXX11STD")), - # CXXFLAGS = R_CMD_config("CXX11FLAGS"), # We don't want the same debug symbols + CXX = paste(sub("^.*ccache", "", R_CMD_config("CXX17")), R_CMD_config("CXX17STD")), + # CXXFLAGS = R_CMD_config("CXX17FLAGS"), # We don't want the same debug symbols LDFLAGS = R_CMD_config("LDFLAGS") ) env_var_list <- with_cloud_support(env_var_list) diff --git a/ruby/red-arrow/ext/arrow/extconf.rb b/ruby/red-arrow/ext/arrow/extconf.rb index 1341d5cbdc386..5bc2dab9563cb 100644 --- a/ruby/red-arrow/ext/arrow/extconf.rb +++ b/ruby/red-arrow/ext/arrow/extconf.rb @@ -38,6 +38,8 @@ end end +$CXXFLAGS += " -std=c++17 " + unless required_pkg_config_package([ "arrow", Arrow::Version::MAJOR,