repo/ci: Shift deb verification -> github (envoyproxy#35579)

Signed-off-by: Ryan Northey <[email protected]>
daixiang0 · Aug 12, 2024 · 5b80c5d · 5b80c5d
1 parent aed9ef2
commit 5b80c5d
Show file tree

Hide file tree

Showing 7 changed files with 177 additions and 187 deletions.
diff --git a/.azure-pipelines/stage/verify.yml b/.azure-pipelines/stage/verify.yml
diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml
@@ -103,13 +103,3 @@ stages:
       runPackaging: variables['RUN_PACKAGING']
       publishDockerhub: variables['PUBLISH_DOCKERHUB']
       publishGithubRelease: variables['PUBLISH_GITHUB_RELEASE']
-
-- stage: verify
-  displayName: Verify
-  dependsOn: ["env", "publish"]
-  variables:
-    RUN_DOCKER: $[stageDependencies.env.repo.outputs['run.docker']]
-  jobs:
-  - template: stage/verify.yml
-    parameters:
-      authGCP: $(GcpServiceAccountKey)
diff --git a/.github/workflows/_stage_publish.yml → .github/workflows/_publish_publish.yml b/.github/workflows/_stage_publish.yml → .github/workflows/_publish_publish.yml
diff --git a/.github/workflows/_publish_verify.yml b/.github/workflows/_publish_verify.yml
@@ -0,0 +1,166 @@
+name: Verify
+
+permissions:
+  contents: read
+
+on:
+  workflow_call:
+    inputs:
+      request:
+        type: string
+        required: true
+      trusted:
+        type: boolean
+        required: true
+
+concurrency:
+  group: >-
+    ${{ github.actor != 'trigger-release-envoy[bot]'
+        && github.event.inputs.head_ref
+        || github.run_id
+    }}-${{ github.event.workflow.id }}-verify
+  cancel-in-progress: true
+
+
+jobs:
+  verify-examples:
+    permissions:
+      contents: read
+      packages: read
+    name: ${{ matrix.name || matrix.target }}
+    uses: ./.github/workflows/_run.yml
+    with:
+      bazel-extra: ${{ matrix.bazel-extra || '--config=rbe-envoy-engflow' }}
+      cache-build-image: ${{ matrix.cache-build-image }}
+      cache-build-image-key-suffix: ${{ matrix.arch == 'arm64' && format('-{0}', matrix.arch) || '' }}
+      container-command: ${{ matrix.container-command }}
+      concurrency-suffix: -${{ matrix.arch || 'x64' }}
+      rbe: ${{ matrix.rbe }}
+      request: ${{ inputs.request }}
+      runs-on: ${{ matrix.runs-on || 'ubuntu-24.04' }}
+      steps-pre: ${{ matrix.steps-pre }}
+      source: ${{ matrix.source }}
+      target: ${{ matrix.target }}
+      trusted: ${{ inputs.trusted }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - name: examples
+          target: verify_examples
+          rbe: false
+          source: |
+            export NO_BUILD_SETUP=1
+          steps-pre: |
+            - run: |
+                # Install expected host packages
+                export DEBIAN_FRONTEND=noninteractive
+                sudo apt-get -qq update -y
+                sudo apt-get -qq install -y --no-install-recommends expect gettext yq whois
+              shell: bash
+            - id: url
+              uses: envoyproxy/toolshed/gh-actions/[email protected]
+              with:
+                options: -Rr
+                input: >-
+                  ${{ inputs.trusted
+                      && fromJSON(inputs.request).request.sha
+                      || fromJSON(inputs.request).request.ref }}
+                filter: |
+                  .[:7] as $sha
+                  | if ${{ inputs.trusted }} then
+                      "envoy-postsubmit"
+                    else
+                      "envoy-pr"
+                    end
+                  | . as $bucket
+                  | "https://storage.googleapis.com/\($bucket)/\($sha)"
+            - uses: envoyproxy/toolshed/gh-actions/docker/[email protected]
+              with:
+                url: %{{ steps.url.outputs.value }}/docker/envoy.tar
+                variant: dev
+            - uses: envoyproxy/toolshed/gh-actions/docker/[email protected]
+              with:
+                url: %{{ steps.url.outputs.value }}/docker/envoy-contrib.tar
+                variant: contrib-dev
+            - uses: envoyproxy/toolshed/gh-actions/docker/[email protected]
+              with:
+                url: %{{ steps.url.outputs.value }}/docker/envoy-google-vrp.tar
+                variant: google-vrp-dev
+            - run: docker images | grep envoy
+              shell: bash
+
+  verify-distro:
+    permissions:
+      contents: read
+      packages: read
+    name: ${{ matrix.name || matrix.target }}
+    uses: ./.github/workflows/_run.yml
+    with:
+      bazel-extra: ${{ matrix.bazel-extra || '--config=rbe-envoy-engflow' }}
+      cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }}
+      cache-build-image-key-suffix: ${{ matrix.arch == 'arm64' && format('-{0}', matrix.arch) || '' }}
+      container-command: ./ci/run_envoy_docker.sh
+      concurrency-suffix: -${{ matrix.arch || 'x64' }}
+      rbe: ${{ matrix.rbe && matrix.rbe || false }}
+      request: ${{ inputs.request }}
+      runs-on: ${{ matrix.runs-on || 'ubuntu-24.04' }}
+      source: |
+        export NO_BUILD_SETUP=1
+        export ENVOY_DOCKER_IN_DOCKER=1
+      target: ${{ matrix.target }}
+      trusted: ${{ inputs.trusted }}
+      steps-pre: |
+        - uses: envoyproxy/toolshed/gh-actions/[email protected]
+          id: url
+          with:
+            options: -Rr
+            input: >-
+              ${{ inputs.trusted
+                  && fromJSON(inputs.request).request.sha
+                  || fromJSON(inputs.request).request.ref }}
+            filter: |
+              .[:7] as $sha
+              | if ${{ inputs.trusted }} then
+                  "envoy-postsubmit"
+                else
+                  "envoy-pr"
+                end
+              | . as $bucket
+              | "https://storage.googleapis.com/\($bucket)/\($sha)/release/release.signed.tar.zst"
+        - uses: envoyproxy/toolshed/gh-actions/[email protected]
+          id: fetch
+          with:
+            url: %{{ steps.url.outputs.value }}
+        - run: |
+            echo ARCH=${{ matrix.arch || 'x64' }} >> $GITHUB_ENV
+            echo DEB_ARCH=${{ matrix.arch != 'arm64' && 'amd64' || 'arm64' }} >> $GITHUB_ENV
+          shell: bash
+        - run: |
+            TEMP_DIR=$(mktemp -d)
+            zstd --stdout -d %{{ steps.fetch.outputs.path }} | tar --warning=no-timestamp -xf - -C "${TEMP_DIR}"
+            mkdir ${TEMP_DIR}/debs
+            tar xf ${TEMP_DIR}/bin/debs.tar.gz -C ${TEMP_DIR}/debs
+            mkdir -p ${TEMP_DIR}/distribution/deb
+            cp -a ${TEMP_DIR}/debs/*_${DEB_ARCH}* ${TEMP_DIR}/distribution/deb
+            cp -a ${TEMP_DIR}/signing.key ${TEMP_DIR}/distribution
+            mkdir -p %{{ runner.temp }}/distribution/${ARCH}
+            tar czf %{{ runner.temp }}/distribution/${ARCH}/packages.${ARCH}.tar.gz -C ${TEMP_DIR}/distribution .
+          shell: bash
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+
+        - name: verify_distro_x64
+          target: verify_distro
+          rbe: true
+
+        - name: verify_distro_arm64
+          target: verify_distro
+          arch: arm64
+          bazel-extra: >-
+            --config=cache-envoy-engflow
+            --config=bes-envoy-engflow
+          runs-on: envoy-arm64-small
diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml
@@ -21,11 +21,16 @@ on:
         default: 75
       cache-build-image:
         type: string
+      cache-build-image-key-suffix:
+        type: string
       catch-errors:
         type: boolean
         default: false
       checkout-extra:
         type: string
+      concurrency-suffix:
+        type: string
+        default:
       container-command:
         type: string
         default: ./ci/run_envoy_docker.sh
@@ -141,7 +146,7 @@ concurrency:
     ${{ github.actor != 'trigger-release-envoy[bot]'
         && github.head_ref
         || github.run_id
-    }}-${{ github.workflow }}-${{ inputs.target }}
+    }}-${{ github.workflow }}-${{ inputs.target }}${{ inputs.concurrency-suffix }}
   cancel-in-progress: true
 
 env:
@@ -190,6 +195,7 @@ jobs:
       uses: envoyproxy/toolshed/gh-actions/docker/cache/[email protected]
       with:
         image_tag: ${{ inputs.cache-build-image }}
+        key-suffix: ${{ inputs.cache-build-image-key-suffix }}
 
     - uses: envoyproxy/toolshed/gh-actions/[email protected]
       id: appauth
@@ -259,11 +265,11 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ inputs.trusted && steps.appauth.outputs.token || github.token }}
         ENVOY_DOCKER_BUILD_DIR: ${{ runner.temp }}
-        ENVOY_RBE: ${{ inputs.rbe != 'false' && 1 || '' }}
+        ENVOY_RBE: ${{ inputs.rbe == true && 1 || '' }}
         RBE_KEY: ${{ secrets.rbe-key }}
         BAZEL_BUILD_EXTRA_OPTIONS: >-
           --config=remote-ci
           ${{ inputs.bazel-extra }}
-          ${{ inputs.rbe != 'false' && format('--jobs={0}', inputs.bazel-rbe-jobs) || '' }}
+          ${{ inputs.rbe == true && format('--jobs={0}', inputs.bazel-rbe-jobs) || '' }}
         BAZEL_FAKE_SCM_REVISION: ${{ github.event_name == 'pull_request' && 'e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9' || '' }}
         CI_TARGET_BRANCH: ${{ fromJSON(inputs.request).request.target-branch }}