diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 29ee7f36ae91..1b9fa0c77703 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,7 +1,7 @@ jobs: - job: BiDiScan pool: - vmImage: 'ubuntu-18.04' + vmImage: 'ubuntu-20.04' steps: - checkout: self @@ -23,7 +23,7 @@ jobs: vmImage: $(imageName) variables: - UBUNTU_CODE_NAME: bionic + UBUNTU_CODE_NAME: focal BOOST_VERSION: boost_1_59_0 BOOST_DIR: $(Pipeline.Workspace)/boost USE_CCACHE: 1 @@ -33,260 +33,268 @@ jobs: CCACHE_CPP2: 1 CCACHE_MAXSIZE: 2G OS_NAME: $(Agent.OS) + PARENT_BRANCH: 5.7 + INVERTED: OFF strategy: matrix: - macOS 10.15 Release: - imageName: 'macOS-10.15' + macOS 12 Release: + imageName: 'macOS-12' Compiler: clang BuildType: RelWithDebInfo ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - macOS 10.15 Debug: - imageName: 'macOS-10.15' + macOS 12 Debug: + imageName: 'macOS-12' Compiler: clang BuildType: Debug # clang-5 and newer compilers - Ubuntu Bionic clang-14 Release: - imageName: 'ubuntu-18.04' + clang-14 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 14 BuildType: RelWithDebInfo ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-14 Release INVERTED=ON: - imageName: 'ubuntu-18.04' + clang-14 Release INVERTED=ON Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 14 BuildType: RelWithDebInfo - Inverted: ON + INVERTED: ON ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-14 Debug: - imageName: 'ubuntu-18.04' + clang-14 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 14 BuildType: Debug - Ubuntu Bionic clang-14 Debug INVERTED=ON: - imageName: 'ubuntu-18.04' + clang-14 Debug INVERTED=ON Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 14 BuildType: Debug - Inverted: ON + INVERTED: ON ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-13 Release: - imageName: 'ubuntu-18.04' + clang-13 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 13 BuildType: RelWithDebInfo - Ubuntu Bionic clang-13 Debug: - imageName: 'ubuntu-18.04' + clang-13 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 13 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-12 Release: - imageName: 'ubuntu-18.04' + clang-12 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 12 BuildType: RelWithDebInfo ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-12 Debug: - imageName: 'ubuntu-18.04' + clang-12 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 12 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-11 Release: - imageName: 'ubuntu-18.04' + clang-11 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 11 BuildType: RelWithDebInfo - Ubuntu Bionic clang-11 Debug: - imageName: 'ubuntu-18.04' + clang-11 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 11 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-10 Release: - imageName: 'ubuntu-18.04' + clang-10 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 10 BuildType: RelWithDebInfo ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-10 Debug: - imageName: 'ubuntu-18.04' + clang-10 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 10 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-9 Release: - imageName: 'ubuntu-18.04' + clang-9 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 9 BuildType: RelWithDebInfo - Ubuntu Bionic clang-9 Debug: - imageName: 'ubuntu-18.04' + clang-9 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 9 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-8 Release: - imageName: 'ubuntu-18.04' + clang-8 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 8 BuildType: RelWithDebInfo ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-8 Debug: - imageName: 'ubuntu-18.04' + clang-8 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 8 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-7 Release: - imageName: 'ubuntu-18.04' + clang-7 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 7 BuildType: RelWithDebInfo - Ubuntu Bionic clang-7 Debug: - imageName: 'ubuntu-18.04' + clang-7 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 7 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-6 Release: - imageName: 'ubuntu-18.04' + clang-6 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 6.0 BuildType: RelWithDebInfo ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-6 Debug: - imageName: 'ubuntu-18.04' + clang-6 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: clang CompilerVer: 6.0 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic clang-5 Release: + clang-5 Release Ubuntu Bionic: imageName: 'ubuntu-18.04' + UBUNTU_CODE_NAME: bionic Compiler: clang CompilerVer: 5.0 BuildType: RelWithDebInfo - Ubuntu Bionic clang-5 Debug: + clang-5 Debug Ubuntu Bionic: imageName: 'ubuntu-18.04' + UBUNTU_CODE_NAME: bionic Compiler: clang CompilerVer: 5.0 BuildType: Debug # gcc-5 and newer compilers - Ubuntu Bionic gcc-11 Release: - imageName: 'ubuntu-18.04' + gcc-11 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: gcc CompilerVer: 11 BuildType: RelWithDebInfo - Ubuntu Bionic gcc-11 Debug: - imageName: 'ubuntu-18.04' + gcc-11 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: gcc CompilerVer: 11 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic gcc-10 Release: - imageName: 'ubuntu-18.04' + gcc-10 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: gcc CompilerVer: 10 BuildType: RelWithDebInfo ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic gcc-10 Debug: - imageName: 'ubuntu-18.04' + gcc-10 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: gcc CompilerVer: 10 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic gcc-9 Release: - imageName: 'ubuntu-18.04' + gcc-9 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: gcc CompilerVer: 9 BuildType: RelWithDebInfo - Ubuntu Bionic gcc-9 Debug: - imageName: 'ubuntu-18.04' + gcc-9 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: gcc CompilerVer: 9 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic gcc-8 Release: - imageName: 'ubuntu-18.04' + gcc-8 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: gcc CompilerVer: 8 BuildType: RelWithDebInfo ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic gcc-8 Debug: - imageName: 'ubuntu-18.04' + gcc-8 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: gcc CompilerVer: 8 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic gcc-7 Release: - imageName: 'ubuntu-18.04' + gcc-7 Release Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: gcc CompilerVer: 7 BuildType: RelWithDebInfo - Ubuntu Bionic gcc-7 Debug: - imageName: 'ubuntu-18.04' + gcc-7 Debug Ubuntu Focal: + imageName: 'ubuntu-20.04' Compiler: gcc CompilerVer: 7 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic gcc-6 Release: + gcc-6 Release Ubuntu Bionic: imageName: 'ubuntu-18.04' + UBUNTU_CODE_NAME: bionic Compiler: gcc CompilerVer: 6 BuildType: RelWithDebInfo ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic gcc-6 Debug: + gcc-6 Debug Ubuntu Bionic: imageName: 'ubuntu-18.04' + UBUNTU_CODE_NAME: bionic Compiler: gcc CompilerVer: 6 BuildType: Debug ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}: - Ubuntu Bionic gcc-5 Release: + gcc-5 Release Ubuntu Bionic: imageName: 'ubuntu-18.04' + UBUNTU_CODE_NAME: bionic Compiler: gcc CompilerVer: 5 BuildType: RelWithDebInfo - Ubuntu Bionic gcc-5 Debug: + gcc-5 Debug Ubuntu Bionic: imageName: 'ubuntu-18.04' + UBUNTU_CODE_NAME: bionic Compiler: gcc CompilerVer: 5 BuildType: Debug @@ -300,22 +308,27 @@ jobs: pwd ls -l if [[ "$OS_NAME" == "Linux" ]]; then - CC=$(Compiler)-$(CompilerVer) + COMPILER_VER=$(CompilerVer) + SELECTED_CC=$(Compiler)-$(CompilerVer) if [[ "$(Compiler)" == "clang" ]]; then - CXX=clang++-$(CompilerVer) - PACKAGES="$CC $PACKAGES" + SELECTED_CXX=clang++-$(CompilerVer) + PACKAGES="$SELECTED_CC $PACKAGES" else - CXX=g++-$(CompilerVer) - PACKAGES="$CXX $PACKAGES" + SELECTED_CXX=g++-$(CompilerVer) + PACKAGES="$SELECTED_CXX $PACKAGES" fi + else + COMPILER_VER=`$(Compiler) -v 2>&1 | head -1 | awk '{print $4}'` + SELECTED_CC=$(Compiler) + SELECTED_CXX=clang++ fi - echo CC=$CC CXX=$CXX BuildType=$(BuildType) Ubuntu=$UBUNTU_CODE_NAME OS_NAME=$OS_NAME + echo SELECTED_CC=$SELECTED_CC SELECTED_CXX=$SELECTED_CXX BuildType=$(BuildType) Ubuntu=$(UBUNTU_CODE_NAME) OS_NAME=$OS_NAME echo --- Configure required LLVM and Ubuntu Toolchain repositories - if [[ "$OS_NAME" == "Linux" ]] && [[ "$CC" == "clang"* ]]; then + if [[ "$OS_NAME" == "Linux" ]] && [[ "$SELECTED_CC" == "clang"* ]]; then PACKAGES="llvm-$(CompilerVer)-dev $PACKAGES" curl -sSL "http://apt.llvm.org/llvm-snapshot.gpg.key" | sudo -E apt-key add - - echo "deb http://apt.llvm.org/$UBUNTU_CODE_NAME/ llvm-toolchain-$UBUNTU_CODE_NAME-$(CompilerVer) main" | sudo tee -a /etc/apt/sources.list > /dev/null + echo "deb http://apt.llvm.org/$(UBUNTU_CODE_NAME)/ llvm-toolchain-$(UBUNTU_CODE_NAME)-$(CompilerVer) main" | sudo tee -a /etc/apt/sources.list > /dev/null fi echo --- Update list of packages and download dependencies @@ -324,43 +337,37 @@ jobs: sudo -E apt-get -yq update >> ~/apt-get-update.log 2>&1 sudo -E apt-get -yq --no-install-suggests --no-install-recommends --allow-unauthenticated install $PACKAGES cmake cmake-curses-gui ccache bison libncurses5-dev libaio-dev libmecab-dev libnuma-dev liblzma-dev libssl-dev libreadline-dev libpam-dev libcurl4-openssl-dev libldap2-dev libkrb5-dev libsasl2-dev libsasl2-modules-gssapi-mit || exit 1; - if [[ "$(Inverted)" != "ON" ]]; then - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --allow-unauthenticated install libevent-dev libeditline-dev liblz4-dev libre2-dev protobuf-compiler libprotobuf-dev libprotoc-dev libicu-dev || exit 1; + if [[ "$(INVERTED)" != "ON" ]]; then + sudo -E apt-get -yq --no-install-suggests --no-install-recommends --allow-unauthenticated install libevent-dev libeditline-dev liblz4-dev protobuf-compiler libprotobuf-dev libprotoc-dev || exit 1; fi else brew update - brew install ccache - brew link ccache - export PATH="/usr/local/opt/ccache/libexec:$PATH" + brew install ccache protobuf lz4 openssl@1.1 fi UPDATE_TIME=$SECONDS echo --- Packages updated in $UPDATE_TIME seconds. - $CC -v - $CXX -v - CCACHE_BIN=$(which ccache) - echo ccache=$CCACHE_BIN + echo "SELECTED_CC=$SELECTED_CC (`which $SELECTED_CC`) SELECTED_CXX=$SELECTED_CXX (`which $SELECTED_CXX`)" + $SELECTED_CC -v + $SELECTED_CXX -v ccache --version ccache -p ccache --zero-stats df -Th - COMPILER_VER=$(CompilerVer) - echo '##vso[task.setvariable variable=CompilerMajorVer]'${COMPILER_VER%.*} - echo '##vso[task.setvariable variable=CC]'$CC - echo '##vso[task.setvariable variable=CXX]'$CXX - echo '##vso[task.setvariable variable=CCACHE_BIN]'$CCACHE_BIN - echo '##vso[task.setvariable variable=UPDATE_TIME]'$UPDATE_TIME - echo '##vso[task.prependpath]/usr/lib/ccache' + echo "##vso[task.setvariable variable=CompilerMajorVer]${COMPILER_VER}" + echo "##vso[task.setvariable variable=SELECTED_CC]$SELECTED_CC" + echo "##vso[task.setvariable variable=SELECTED_CXX]$SELECTED_CXX" + echo "##vso[task.setvariable variable=UPDATE_TIME]$UPDATE_TIME" displayName: '*** Install Build Dependencies' - task: Cache@2 continueOnError: true inputs: - key: ccache | $(Agent.OS)-$(Compiler)-$(CompilerMajorVer)-$(BuildType) | $(Build.SourceVersion) - restoreKeys: ccache | $(Agent.OS)-$(Compiler)-$(CompilerMajorVer)-$(BuildType) + key: '"ccache"| "$(PARENT_BRANCH)" | "$(imageName)-$(Compiler)-$(CompilerMajorVer)-$(BuildType)" | "$(Build.SourceVersion)"' + restoreKeys: '"ccache" | "$(PARENT_BRANCH)" | "$(imageName)-$(Compiler)-$(CompilerMajorVer)-$(BuildType)"' path: $(CCACHE_DIR) displayName: '*** Download/upload ccached files' @@ -383,6 +390,7 @@ jobs: - script: | df -Th + echo SELECTED_CC=$SELECTED_CC SELECTED_CXX=$SELECTED_CXX BuildType=$(BuildType) INVERTED=$(INVERTED) imageName=$(imageName) OS_NAME=$OS_NAME echo --- Set cmake parameters COMPILE_OPT+=( -DCMAKE_C_FLAGS_DEBUG=-g1 @@ -391,13 +399,13 @@ jobs: '-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=-O2 -g1 -DNDEBUG' ) - if [[ "$CC" =~ clang-(5.0|6.0|7|8)$ ]]; then + if [[ "$OS_NAME" == "Linux" ]] && [[ "$SELECTED_CC" =~ clang-(5.0)$ ]]; then COMPILE_OPT+=( '-DCMAKE_C_FLAGS=-isystem /usr/include/c++/9 -isystem /usr/include' '-DCMAKE_CXX_FLAGS=-isystem /usr/include/c++/9 -isystem /usr/include' ) fi - if [[ "$CC" == "clang"* ]]; then + if [[ "$OS_NAME" == "Linux" ]] && [[ "$SELECTED_CC" == "clang"* ]]; then COMPILE_OPT+=( '-DCMAKE_AR=/usr/bin/ar' '-DCMAKE_RANLIB=/usr/bin/ranlib' @@ -410,8 +418,10 @@ jobs: -DENABLE_DOWNLOADS=1 -DDOWNLOAD_BOOST=1 -DWITH_BOOST=$(BOOST_DIR) - -DCMAKE_C_COMPILER_LAUNCHER=$CCACHE_BIN - -DCMAKE_CXX_COMPILER_LAUNCHER=$CCACHE_BIN + -DCMAKE_C_COMPILER=$SELECTED_CC + -DCMAKE_CXX_COMPILER=$SELECTED_CXX + -DCMAKE_C_COMPILER_LAUNCHER=ccache + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DWITH_KEYRING_VAULT=ON -DWITH_KEYRING_VAULT_TEST=ON -DWITH_PAM=ON @@ -423,6 +433,8 @@ jobs: -DWITH_ROCKSDB=OFF -DWITH_TOKUDB=OFF -DWITH_PROTOBUF=bundled + -DWITH_SSL=/usr/local/opt/openssl@1.1 + -DWITH_ZLIB=bundled " else CMAKE_OPT+=" @@ -431,25 +443,21 @@ jobs: -DWITH_TOKUDB=ON -DWITH_CURL=system -DWITH_MECAB=system - -DWITH_RAPIDJSON=bundled -DWITH_SSL=system -DWITH_LIBEVENT=bundled + -DWITH_PROTOBUF=bundled " - if [[ "$(Inverted)" != "ON" ]]; then + if [[ "$(INVERTED)" != "ON" ]]; then CMAKE_OPT+=" -DWITH_READLINE=system - -DWITH_ICU=system -DWITH_LZ4=bundled - -DWITH_PROTOBUF=system -DWITH_ZLIB=system -DWITH_NUMA=ON " else CMAKE_OPT+=" -DWITH_EDITLINE=bundled - -DWITH_ICU=bundled -DWITH_LZ4=bundled - -DWITH_PROTOBUF=bundled -DWITH_ZLIB=bundled -DWITH_NUMA=OFF -DWITH_ARCHIVE_STORAGE_ENGINE=OFF @@ -465,7 +473,7 @@ jobs: echo --- CMAKE_OPT=\"$CMAKE_OPT\" echo --- COMPILE_OPT=\"${COMPILE_OPT[@]}\" mkdir bin; cd bin - CC=$CC CXX=$CXX cmake .. $CMAKE_OPT "${COMPILE_OPT[@]}" || exit 1 + cmake .. $CMAKE_OPT "${COMPILE_OPT[@]}" || exit 1 rm -f $(BOOST_DIR)/$(BOOST_VERSION).tar.gz CMAKE_TIME=$SECONDS @@ -473,7 +481,7 @@ jobs: echo '##vso[task.setvariable variable=CMAKE_TIME]'$CMAKE_TIME - displayName: '*** CC=$(Compiler)-$(CompilerVer) cmake .. -DCMAKE_BUILD_TYPE=$(BuildType)' + displayName: '*** cmake -DCMAKE_BUILD_TYPE=$(BuildType)' - script: | df -Th