sync from master.

.azure-pipelines/bazel.yml

@@ -3,6 +3,20 @@ parameters:
     displayName: "CI target"
     type: string
     default: bazel.release
+  - name: artifactSuffix
+    displayName: "Suffix of artifact"
+    type: string
+    default: ""
+  - name: rbe
+    displayName: "Enable RBE"
+    type: boolean
+    default: true
+  - name: managedAgent
+    type: boolean
+    default: true
+  - name: bazelBuildExtraOptions
+    type: string
+    default: ""
 
 steps:
   - task: Cache@2
@@ -13,6 +27,7 @@ steps:
 
   - bash: .azure-pipelines/cleanup.sh
     displayName: "Removing tools from agent"
+    condition: ${{ parameters.managedAgent }}
 
   - bash: |
       echo "disk space at beginning of build:"
@@ -27,18 +42,22 @@ steps:
       }' | sudo tee /etc/docker/daemon.json
       sudo service docker restart
     displayName: "Enable IPv6"
+    condition: ${{ parameters.managedAgent }}
 
   - script: ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ parameters.ciTarget }}'
     workingDirectory: $(Build.SourcesDirectory)
     env:
       ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-      ENVOY_RBE: "true"
-      # Use https://docs.bazel.build/versions/master/command-line-reference.html#flag--experimental_repository_cache_hardlinks
-      # to save disk space.
-      BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs) --curses=no --experimental_repository_cache_hardlinks"
-      BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
-      BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
-      GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+      ${{ if parameters.rbe }}:
+        ENVOY_RBE: "1"
+        BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs) ${{ parameters.bazelBuildExtraOptions }}"
+        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+      ${{ if eq(parameters.rbe, false) }}:
+        BAZEL_BUILD_EXTRA_OPTIONS: "${{ parameters.bazelBuildExtraOptions }}"
+        BAZEL_REMOTE_CACHE: $(LocalBuildCache)
+
     displayName: "Run CI script"
 
   - bash: |
@@ -57,5 +76,5 @@ steps:
   - task: PublishBuildArtifacts@1
     inputs:
       pathtoPublish: "$(Build.StagingDirectory)/envoy"
-      artifactName: ${{ parameters.ciTarget }}
+      artifactName: ${{ parameters.ciTarget }}${{ parameters.artifactSuffix }}
     condition: always()

.azure-pipelines/cleanup.sh

@@ -3,7 +3,7 @@
 set -e
 
 # Temporary script to remove tools from Azure pipelines agent to create more disk space room.
-
-sudo apt-get purge -y 'ghc-*' 'zulu-*-azure-jdk' 'libllvm*' 'mysql-*' 'dotnet-*' 'cpp-*'
+sudo apt-get update -y
+sudo apt-get purge -y --no-upgrade 'ghc-*' 'zulu-*-azure-jdk' 'libllvm*' 'mysql-*' 'dotnet-*' 'libgl1'
 
 dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -rn

.azure-pipelines/pipelines.yml

@@ -14,7 +14,7 @@ jobs:
   - job: format
     dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
     pool:
-      vmImage: "ubuntu-16.04"
+      vmImage: "ubuntu-18.04"
     steps:
       - task: Cache@2
         inputs:
@@ -44,12 +44,28 @@ jobs:
     condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest')))
     timeoutInMinutes: 360
     pool:
-      vmImage: "ubuntu-16.04"
+      vmImage: "ubuntu-18.04"
     steps:
       - template: bazel.yml
         parameters:
           ciTarget: bazel.release
 
+  - job: release_arm64
+    displayName: "Linux-arm64 release"
+    dependsOn: ["format"]
+    # For master builds, continue even if format fails
+    condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest')))
+    timeoutInMinutes: 360
+    pool: "arm-large"
+    steps:
+      - template: bazel.yml
+        parameters:
+          managedAgent: false
+          ciTarget: bazel.release
+          rbe: false
+          artifactSuffix: ".arm64"
+          bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base"
+
   - job: bazel
     displayName: "Linux-x64"
     dependsOn: ["release"]
@@ -70,18 +86,47 @@ jobs:
           CI_TARGET: "bazel.compile_time_options"
     timeoutInMinutes: 360
     pool:
-      vmImage: "Ubuntu 16.04"
+      vmImage: "ubuntu-18.04"
     steps:
       - template: bazel.yml
         parameters:
           ciTarget: $(CI_TARGET)
 
-  - job: docker
-    displayName: "Linux-x64 docker"
+  - job: coverage
+    displayName: "Linux-x64"
     dependsOn: ["release"]
+    timeoutInMinutes: 360
+    pool: "x64-large"
+    strategy:
+      maxParallel: 2
+      matrix:
+        coverage:
+          CI_TARGET: "coverage"
+        fuzz_coverage:
+          CI_TARGET: "fuzz_coverage"
+    steps:
+      - template: bazel.yml
+        parameters:
+          managedAgent: false
+          ciTarget: bazel.$(CI_TARGET)
+          rbe: false
+          # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces
+          bazelBuildExtraOptions: "--define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base"
+
+      - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)'
+        displayName: "Upload $(CI_TARGET) Report to GCS"
+        env:
+          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+          GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+          GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)
+        condition: always()
+
+  - job: docker
+    displayName: "Linux multi-arch docker"
+    dependsOn: ["release","release_arm64"]
     condition: and(succeeded(), eq(variables['PostSubmit'], 'true'), ne(variables['Build.Reason'], 'PullRequest'))
     pool:
-      vmImage: "ubuntu-16.04"
+      vmImage: "ubuntu-18.04"
     steps:
       - task: DownloadBuildArtifacts@0
         inputs:
@@ -90,10 +135,17 @@ jobs:
           itemPattern: "bazel.release/envoy_binary.tar.gz"
           downloadType: single
           targetPath: $(Build.StagingDirectory)
-
+      - task: DownloadBuildArtifacts@0
+        inputs:
+          buildType: current
+          artifactName: "bazel.release.arm64"
+          itemPattern: "bazel.release.arm64/envoy_binary.tar.gz"
+          downloadType: single
+          targetPath: $(Build.StagingDirectory)
       - bash: |
           set -e
-          tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz
+          mkdir -p linux/amd64 && tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz -C ./linux/amd64
+          mkdir -p linux/arm64 && tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz -C ./linux/arm64
           ci/docker_ci.sh
         workingDirectory: $(Build.SourcesDirectory)
         env:
@@ -114,6 +166,7 @@ jobs:
       - script: ./ci/mac_ci_steps.sh
         displayName: "Run Mac CI"
         env:
+          BAZEL_BUILD_EXTRA_OPTIONS: --remote_download_toplevel
           BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
           BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
           GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
@@ -130,24 +183,11 @@ jobs:
     pool:
       vmImage: "windows-latest"
     steps:
-      - powershell: |
-          .\ci\windows_ci_setup.ps1
-          Write-Host "##vso[task.prependpath]$env:TOOLS_BIN_DIR\usr\bin"
-          Write-Host "##vso[task.prependpath]$env:VC_TOOLS_BIN_X64;$env:VC_CMAKE_PATH\CMake\bin;$env:VC_CMAKE_PATH\Ninja"
-          Write-Host "##vso[task.prependpath]$env:TOOLS_BIN_DIR"
-        displayName: "Install dependencies"
-        env:
-          TOOLS_BIN_DIR: $(Pipeline.Workspace)\bin
-          VC_CMAKE_PATH: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake"
-          VC_TOOLS_BIN_X64: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC\\Tools\\MSVC\\14.25.28610\\bin\\HostX64\\x64"
-
-      - bash: ci/windows_ci_steps.sh
+      - bash: ci/run_envoy_docker_windows.sh ci/windows_ci_steps.sh
         displayName: "Run Windows CI"
         env:
-          BAZEL_VC: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC"
-          BAZEL_SH: $(Pipeline.Workspace)/bin/usr/bin/bash.exe
+          ENVOY_RBE: "true"
+          BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs)"
           BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
           BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
           GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
-          MSYS2_ARG_CONV_EXCL: "*"
-          TMPDIR: $(Agent.TempDirectory)

.bazelci/presubmit.yml

@@ -1,14 +1,30 @@
 ---
 tasks:
   gcc:
+    name: "GCC"
     platform: ubuntu1804
     build_targets:
       - "//source/exe:envoy-static"
   rbe:
+    name: "RBE"
     platform: ubuntu1804
     test_targets:
-      - "//test/..."
+      - "//test/common/common/..."
+      - "//test/integration/..."
+      - "//test/exe/..."
     test_flags:
-      - "--config=remote-clang"
+      - "--config=remote-clang-libc++"
       - "--config=remote-ci"
       - "--jobs=75"
+  coverage:
+    name: "Coverage"
+    platform: ubuntu1804
+    shell_commands:
+      - "bazel/setup_clang.sh /usr/lib/llvm-10"
+    test_targets:
+      - "//test/common/common/..."
+      - "//test/integration/..."
+      - "//test/exe/..."
+    test_flags:
+      - "--config=coverage"
+      - "--config=clang"