Merge remote-tracking branch 'upstream/main' into add_filter_states

kyessenov · Sep 11, 2023 · 90e251a · 90e251a
2 parents ce60e4e + 154219f
commit 90e251a
Show file tree

Hide file tree

Showing 292 changed files with 7,608 additions and 2,172 deletions.
diff --git a/.azure-pipelines/bazel.yml b/.azure-pipelines/bazel.yml
@@ -2,7 +2,7 @@ parameters:
 - name: ciTarget
   displayName: "CI target"
   type: string
-  default: bazel.release
+  default: release
 - name: artifactSuffix
   displayName: "Suffix of artifact"
   type: string
@@ -11,13 +11,16 @@ parameters:
 # caching
 - name: cacheKeyDocker
   type: string
-  default: ".devcontainer/Dockerfile"
+  default: ".bazelrc"
 - name: cacheKeyVersion
   type: string
   default: $(cacheKeyVersion)
 - name: pathCacheTemp
   type: string
   default: $(pathCacheTemp)
+- name: cacheName
+  type: string
+  default:
 
 - name: tmpfsCacheDisabled
   type: string
@@ -145,19 +148,25 @@ steps:
 - bash: |
     set -e
     CACHE_DIRS=(
+        "$(Build.StagingDirectory)/envoy"
         "$(Build.StagingDirectory)/.cache/"
         "$(Build.StagingDirectory)/bazel_root/install/"
         "$(Build.StagingDirectory)/repository_cache/"
         "$(Build.StagingDirectory)/bazel_root/base/external")
     sudo mkdir -p "${CACHE_DIRS[@]}"
-    sudo chown -R vsts:vsts "${CACHE_DIRS[@]}" $(Build.StagingDirectory)/bazel_root/
-    echo "Created bazel cache directories: "${CACHE_DIRS[*]}""
+    if id -u vsts &> /dev/null; then
+        sudo chown -R vsts:vsts "${CACHE_DIRS[@]}" $(Build.StagingDirectory)/bazel_root/
+    else
+        sudo chown -R azure-pipelines:azure-pipelines "${CACHE_DIRS[@]}" $(Build.StagingDirectory)/bazel_root/
+    fi
+    echo "Created bazel directories: "${CACHE_DIRS[*]}""
   displayName: "Create bazel directories"
-  condition: and(succeeded(), eq('${{ parameters.managedAgent }}', true), eq('${{ parameters.tmpfsDockerDisabled }}', true))
+  condition: and(succeeded(), eq('${{ parameters.tmpfsDockerDisabled }}', true))
 
 # Caching
 - template: cached.yml
   parameters:
+    cacheName: "${{ parameters.cacheName }}"
     keyBazel: "${{ parameters.cacheKeyBazel }}"
     keyDocker: "${{ parameters.cacheKeyDocker }}"
     pathDockerBind: "${{ parameters.pathDockerBind }}"
@@ -166,13 +175,50 @@ steps:
     tmpfsDisabled: "${{ parameters.tmpfsCacheDisabled }}"
     tmpfsDockerDisabled: "${{ parameters.tmpfsDockerDisabled }}"
 
+- script: |
+    if [[ "${{ parameters.bazelUseBES }}" == 'false' ]]; then
+        unset GOOGLE_BES_PROJECT_ID
+    fi
+    ci/run_envoy_docker.sh 'ci/do_ci.sh fetch-${{ parameters.ciTarget }}'
+  condition: and(not(canceled()), not(failed()), ne('${{ parameters.cacheName }}', ''), ne(variables.CACHE_RESTORED, 'true'))
+  workingDirectory: $(Build.SourcesDirectory)
+  env:
+    ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+    GITHUB_TOKEN: "${{ parameters.authGithub }}"
+    BAZEL_STARTUP_EXTRA_OPTIONS: "${{ parameters.bazelStartupExtraOptions }}"
+    ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+      CI_TARGET_BRANCH: "origin/$(System.PullRequest.TargetBranch)"
+    ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+      CI_TARGET_BRANCH: "origin/$(Build.SourceBranchName)"
+    # Any PR or CI run in envoy-presubmit uses the fake SCM hash
+    ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'envoy-presubmit')) }}:
+      # sha1sum of `ENVOY_PULL_REQUEST`
+      BAZEL_FAKE_SCM_REVISION: e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9
+    ${{ if parameters.rbe }}:
+      GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+      ENVOY_RBE: "1"
+      BAZEL_BUILD_EXTRA_OPTIONS: "${{ parameters.bazelConfigRBE }} ${{ parameters.bazelBuildExtraOptions }}"
+    ${{ if eq(parameters.rbe, false) }}:
+      BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci ${{ parameters.bazelBuildExtraOptions }}"
+      BAZEL_REMOTE_CACHE: $(LocalBuildCache)
+    ${{ each var in parameters.env }}:
+      ${{ var.key }}: ${{ var.value }}
+  displayName: "Fetch assets (${{ parameters.ciTarget }})"
+
 - ${{ each step in parameters.stepsPre }}:
   - ${{ each pair in step }}:
       ${{ pair.key }}: ${{ pair.value }}
 
 - bash: |
     echo "disk space at beginning of build:"
     df -h
+    if [[ -e "$(Build.StagingDirectory)/bazel_root/base/external" ]]; then
+        du -sh "$(Build.StagingDirectory)/bazel_root/base/external"
+    fi
+    if [[ -e "$(Build.StagingDirectory)/repository_cache" ]]; then
+        du -sh "$(Build.StagingDirectory)/repository_cache"
+    fi
+
   displayName: "Check disk space at beginning"
 
 - bash: |
@@ -228,6 +274,9 @@ steps:
         cp -a $hprof $(Build.StagingDirectory)/envoy/hprof
     done
 
+    du -sh "$(Build.StagingDirectory)"/bazel_root/base/external
+    du -sh "$(Build.StagingDirectory)"/repository_cache
+
     cp -a "$(Build.StagingDirectory)/bazel_root/base/server/jvm.out" $(Build.StagingDirectory)/envoy
 
     if [[ "${{ parameters.artifactSuffix }}" == ".arm64" ]]; then
@@ -247,6 +296,18 @@ steps:
   - ${{ each pair in step }}:
       ${{ pair.key }}: ${{ pair.value }}
 
+- script: |
+    set -e
+    sudo .azure-pipelines/docker/save_cache.sh "$(Build.StagingDirectory)" /mnt/cache/all true true
+    if id -u vsts &> /dev/null; then
+        sudo chown -R vsts:vsts /mnt/cache/all
+    else
+        sudo chown -R azure-pipelines:azure-pipelines /mnt/cache/all
+    fi
+
+  displayName: "Cache/save (${{ parameters.cacheName}})"
+  condition: and(succeeded(), ne('${{ parameters.cacheName }}', ''), ne(variables.CACHE_RESTORED, 'true'))
+
 - task: PublishTestResults@2
   inputs:
     publishRunAttachments: false

diff --git a/.azure-pipelines/cached.yml b/.azure-pipelines/cached.yml
@@ -1,14 +1,14 @@
 
 parameters:
-- name: name
-  type: string
-  default: $(cacheKeyName)
 - name: arch
   type: string
   default: ""
 - name: version
   type: string
   default: $(cacheKeyVersion)
+- name: cacheName
+  type: string
+  default:
 
 - name: keyDocker
   type: string
@@ -45,20 +45,32 @@ steps:
   displayName: "Cache/prepare"
 
 - task: Cache@2
+  condition: and(not(canceled()), ne('${{ parameters.cacheName }}', ''))
+  env:
+    VSO_DEDUP_REDIRECT_TIMEOUT_IN_SEC: "${{ parameters.cacheTimeoutWorkaround }}"
+  displayName: "Cache (${{ parameters.cacheName }})"
+  inputs:
+    key: '${{ parameters.cacheName }} | "${{ parameters.version }}" | "${{ parameters.arch }}" | ${{ parameters.keyDocker }} | ${{ parameters.keyBazel }}'
+    path: "${{ parameters.pathTemp }}/all"
+    cacheHitVar: CACHE_RESTORED
+
+- task: Cache@2
+  condition: and(not(canceled()), not(failed()), or(ne(variables.CACHE_RESTORED, 'true'), eq('${{ parameters.cacheName }}', '')))
   env:
     VSO_DEDUP_REDIRECT_TIMEOUT_IN_SEC: "${{ parameters.cacheTimeoutWorkaround }}"
   displayName: "Cache (Docker)"
   inputs:
-    key: '${{ parameters.name }} | "${{ parameters.version }}" | "${{ parameters.arch }}" | ${{ parameters.keyDocker }}'
+    key: '"${{ parameters.version }}" | "${{ parameters.arch }}" | ${{ parameters.keyDocker }} | docker'
     path: "${{ parameters.pathTemp }}/docker"
     cacheHitVar: DOCKER_CACHE_RESTORED
 
 - task: Cache@2
+  condition: and(not(canceled()), not(failed()), or(ne(variables.CACHE_RESTORED, 'true'), eq('${{ parameters.cacheName }}', '')))
   env:
     VSO_DEDUP_REDIRECT_TIMEOUT_IN_SEC: "${{ parameters.cacheTimeoutWorkaround }}"
   displayName: "Cache (Bazel)"
   inputs:
-    key: '${{ parameters.name }} | bazel | "${{ parameters.version }}" | "${{ parameters.arch }}" | ${{ parameters.keyBazel }}'
+    key: '"${{ parameters.version }}" | "${{ parameters.arch }}" | ${{ parameters.keyBazel }} | bazel'
     path: "${{ parameters.pathTemp }}/bazel"
     cacheHitVar: BAZEL_CACHE_RESTORED
 
@@ -67,9 +79,9 @@ steps:
   env:
     DOCKER_RESTORED: $(DOCKER_CACHE_RESTORED)
     BAZEL_RESTORED: $(BAZEL_CACHE_RESTORED)
-  displayName: "Cache/prime (${{ parameters.name }})"
+  displayName: "Cache/prime (Docker/Bazel)"
   # TODO(phlax): figure if there is a way to test cache without downloading it
-  condition: and(not(canceled()), eq(${{ parameters.prime }}, true), or(ne(variables.DOCKER_CACHE_RESTORED, 'true'), ne(variables.BAZEL_CACHE_RESTORED, 'true')))
+  condition: and(not(canceled()), eq(${{ parameters.prime }}, true), eq('${{ parameters.cacheName }}', ''), or(ne(variables.DOCKER_CACHE_RESTORED, 'true'), ne(variables.BAZEL_CACHE_RESTORED, 'true')))
 
 # Load the caches for a job
 - script: sudo .azure-pipelines/docker/load_caches.sh "$(Build.StagingDirectory)" "${{ parameters.pathTemp }}" "${{ parameters.pathDockerBind }}" "${{ parameters.tmpfsDockerDisabled }}"

diff --git a/.azure-pipelines/docker/create_cache.sh b/.azure-pipelines/docker/create_cache.sh
@@ -3,7 +3,8 @@
 set -o pipefail
 
 CACHE_TARBALL="${1}"
-shift
+ROOT_DIR="${2}"
+shift 2
 
 echo "Exporting ${*} -> ${CACHE_TARBALL}"
 
@@ -12,9 +13,15 @@ mkdir -p "$CACHE_PATH"
 
 CACHE_ARGS=()
 for path in "$@"; do
-    total="$(du -sh "$path" | cut -f1)"
-    echo "Adding cache dir (${path}): ${total}"
-    CACHE_ARGS+=(-C "$path" .)
+    if [[ "$ROOT_DIR" == "." ]]; then
+        total="$(du -sh "$path" | cut -f1)"
+        echo "Adding cache dir (${path}): ${total}"
+        CACHE_ARGS+=(-C "$path" .)
+    else
+        total="$(du -sh "${ROOT_DIR}/$path" | cut -f1)"
+        echo "Adding cache dir (${ROOT_DIR}/${path}): ${total}"
+        CACHE_ARGS+=(-C "$ROOT_DIR" "$path")
+    fi
 done
 
 tar cf - "${CACHE_ARGS[@]}" | zstd - -q -T0 -o "$CACHE_TARBALL"

diff --git a/.azure-pipelines/docker/load_caches.sh b/.azure-pipelines/docker/load_caches.sh
@@ -11,10 +11,15 @@ if [[ -z "$CACHE_PATH" ]]; then
     exit 1
 fi
 
-DOCKER_CACHE_PATH="${CACHE_PATH}/docker"
-DOCKER_CACHE_TARBALL="${DOCKER_CACHE_PATH}/docker.tar.zst"
+if [[ -e "${CACHE_PATH}/all" ]]; then
+    DOCKER_CACHE_PATH="${CACHE_PATH}/all"
+    BAZEL_CACHE_PATH="${CACHE_PATH}/all"
+else
+    DOCKER_CACHE_PATH="${CACHE_PATH}/docker"
+    BAZEL_CACHE_PATH="${CACHE_PATH}/bazel"
+fi
 
-BAZEL_CACHE_PATH="${CACHE_PATH}/bazel"
+DOCKER_CACHE_TARBALL="${DOCKER_CACHE_PATH}/docker.tar.zst"
 BAZEL_CACHE_TARBALL="${BAZEL_CACHE_PATH}/bazel.tar.zst"
 
 
@@ -42,9 +47,7 @@ remount_docker () {
 extract_docker () {
     if [[ -e "${DOCKER_CACHE_TARBALL}" ]]; then
         echo "Extracting docker cache ${DOCKER_CACHE_TARBALL} -> /var/lib/docker ..."
-        ls -alh "$DOCKER_CACHE_TARBALL"
         zstd --stdout -d "$DOCKER_CACHE_TARBALL" | tar --warning=no-timestamp -xf - -C /var/lib/docker
-        touch /tmp/DOCKER_CACHE_RESTORED
     else
         echo "No Docker cache to restore, starting Docker with no data"
     fi
@@ -54,6 +57,11 @@ extract_bazel () {
     if [[ -e "${BAZEL_CACHE_TARBALL}" ]]; then
         echo "Extracting bazel cache ${BAZEL_CACHE_TARBALL} -> ${ENVOY_DOCKER_BUILD_DIR} ..."
         zstd --stdout -d "$BAZEL_CACHE_TARBALL" | tar --warning=no-timestamp -xf - -C "${ENVOY_DOCKER_BUILD_DIR}"
+        if id -u vsts &> /dev/null; then
+            sudo chown -R vsts:vsts "${ENVOY_DOCKER_BUILD_DIR}"
+        else
+            sudo chown -R azure-pipelines:azure-pipelines "${ENVOY_DOCKER_BUILD_DIR}"
+        fi
     else
         echo "No bazel cache to restore, starting bazel with no data"
     fi

diff --git a/.azure-pipelines/docker/prepare_cache.sh b/.azure-pipelines/docker/prepare_cache.sh
@@ -3,7 +3,6 @@
 DOCKER_CACHE_PATH="$1"
 NO_MOUNT_TMPFS="${2:-}"
 DOCKER_CACHE_OWNERSHIP="vsts:vsts"
-TMPFS_SIZE=5G
 
 if [[ -z "$DOCKER_CACHE_PATH" ]]; then
     echo "prepare_docker_cache called without path arg" >&2
@@ -14,6 +13,14 @@ if ! id -u vsts &> /dev/null; then
     DOCKER_CACHE_OWNERSHIP=azure-pipelines
 fi
 
+tmpfs_size () {
+    # Make this 2/3 of total memory
+    total_mem="$(grep MemTotal /proc/meminfo  | cut -d' ' -f2- | xargs | cut -d' ' -f1)"
+    bc <<< "$total_mem"*2/3*1024
+}
+
+TMPFS_SIZE="$(tmpfs_size)"
+
 echo "Creating cache directory (${DOCKER_CACHE_PATH}) ..."
 mkdir -p "${DOCKER_CACHE_PATH}"
 if [[ -z "$NO_MOUNT_TMPFS" ]]; then

diff --git a/.azure-pipelines/docker/prime_cache.sh b/.azure-pipelines/docker/prime_cache.sh
@@ -33,6 +33,18 @@ fi
 echo "==================================================="
 echo
 
+echo
+echo "================ Docker fetch ======================"
+if [[ "$DOCKER_RESTORED" != "true" ]]; then
+    echo "Fetching Docker"
+    ./ci/run_envoy_docker.sh uname -a
+    docker images
+else
+    echo "Not fetching Docker as it was restored"
+fi
+echo "==================================================="
+echo
+
 echo
 echo "================ Bazel fetch ======================"
 # Fetch bazel dependencies
@@ -45,7 +57,6 @@ fi
 echo "==================================================="
 echo
 
-docker images
 df -h
 
 echo
@@ -54,11 +65,11 @@ echo "================ Save caches ======================"
 if [[ "$DOCKER_RESTORED" != "true" ]]; then
     echo "Stopping docker"
     sudo systemctl stop docker docker.socket
-    sudo ./.azure-pipelines/docker/create_cache.sh "${DOCKER_CACHE_TARBALL}" /var/lib/docker
+    sudo ./.azure-pipelines/docker/create_cache.sh "${DOCKER_CACHE_TARBALL}" . /var/lib/docker
 fi
 
 if [[ "$BAZEL_RESTORED" != "true" ]]; then
-    sudo ./.azure-pipelines/docker/create_cache.sh "${BAZEL_CACHE_TARBALL}" "${BAZEL_PATH}"
+    sudo ./.azure-pipelines/docker/create_cache.sh "${BAZEL_CACHE_TARBALL}" . "${BAZEL_PATH}"
 fi
 sudo chmod o+r -R "${CACHE_PATH}"
 echo "==================================================="

diff --git a/.azure-pipelines/docker/save_cache.sh b/.azure-pipelines/docker/save_cache.sh
@@ -1,37 +1,42 @@
 #!/bin/bash -e
 
 set -o pipefail
+ENVOY_DOCKER_BUILD_DIR="$1"
+CACHE_PATH="$2"
+NO_MOUNT_TMPFS="${3:-}"
+CACHE_BAZEL="${4:-}"
 
-DOCKER_CACHE_PATH="$1"
-NO_MOUNT_TMPFS="${2:-}"
-
-if [[ -z "$DOCKER_CACHE_PATH" ]]; then
+if [[ -z "$CACHE_PATH" ]]; then
     echo "prime_docker_cache called without path arg" >&2
     exit 1
 fi
 
-if [[ -e /tmp/DOCKER_CACHE_RESTORED ]]; then
-    echo "Not saving cache as it was restored"
-    exit 0
-fi
-
-DOCKER_CACHE_TARBALL="${DOCKER_CACHE_PATH}/docker.tar.zst"
+DOCKER_CACHE_TARBALL="${CACHE_PATH}/docker.tar.zst"
+BAZEL_CACHE_TARBALL="${CACHE_PATH}/bazel.tar.zst"
 
 docker images
 
 echo "Stopping Docker ..."
-systemctl stop docker
+systemctl stop docker docker.socket
 
-echo "Creating directory to save tarball: ${DOCKER_CACHE_PATH}"
-mkdir -p "$DOCKER_CACHE_PATH"
+echo "Creating directory to save tarball: ${CACHE_PATH}"
+mkdir -p "$CACHE_PATH"
 
 if [[ -z "$NO_MOUNT_TMPFS" ]]; then
-    echo "Mount tmpfs directory: ${DOCKER_CACHE_PATH}"
-    mount -t tmpfs none "$DOCKER_CACHE_PATH"
+    echo "Mount tmpfs directory: ${CACHE_PATH}"
+    mount -t tmpfs none "$CACHE_PATH"
 fi
 
-echo "Creating tarball: /var/lib/docker -> ${DOCKER_CACHE_TARBALL}"
-tar cf - -C /var/lib/docker . | zstd - -T0 -o "$DOCKER_CACHE_TARBALL"
-
-echo "Docker cache tarball created: ${DOCKER_CACHE_TARBALL}"
-ls -lh "$DOCKER_CACHE_TARBALL"
+./.azure-pipelines/docker/create_cache.sh \
+    "${DOCKER_CACHE_TARBALL}" \
+    . \
+    /var/lib/docker
+
+if [[ "$CACHE_BAZEL" == "true" ]]; then
+    ./.azure-pipelines/docker/create_cache.sh \
+        "${BAZEL_CACHE_TARBALL}" \
+        "${ENVOY_DOCKER_BUILD_DIR}" \
+        bazel_root/install \
+        bazel_root/base/external \
+        repository_cache
+fi