From 4b8eeb73f403cd877fb60fa9878f3d08fd29dbd4 Mon Sep 17 00:00:00 2001 From: phlax Date: Sat, 9 Dec 2023 09:56:46 +0000 Subject: [PATCH] github/ci: New CI framework (#30921) Signed-off-by: Ryan Northey --- .github/actions/env/action.yml | 196 -------- .../actions/verify/examples/setup/action.yml | 37 -- .github/config.yml | 253 ++++++++++ .github/workflows/README.md | 198 ++++++++ .github/workflows/_cache.yml | 69 +++ .github/workflows/_cache_docker.yml | 43 -- .github/workflows/_ci.yml | 191 -------- .github/workflows/_env.yml | 192 -------- .github/workflows/_finish.yml | 118 +++++ .github/workflows/_load.yml | 153 ++++++ .github/workflows/_mobile_container_ci.yml | 147 ++++++ .github/workflows/_precheck_deps.yml | 43 +- .github/workflows/_request.yml | 121 +++++ .github/workflows/_run.yml | 261 ++++++++++ .github/workflows/_stage_publish.yml | 130 +++-- .github/workflows/_stage_verify.yml | 73 +-- .github/workflows/_start.yml | 131 +++++ .github/workflows/_workflow-start.yml | 51 -- .github/workflows/codeql-push.yml | 2 +- .github/workflows/command.yml | 45 ++ .github/workflows/commands.yml | 31 -- .github/workflows/envoy-dependency.yml | 68 +-- .github/workflows/envoy-macos.yml | 69 ++- .github/workflows/envoy-prechecks.yml | 74 ++- .github/workflows/envoy-publish.yml | 97 ++-- .github/workflows/envoy-release.yml | 24 +- .github/workflows/envoy-sync.yml | 10 +- .github/workflows/envoy-windows.yml | 95 ++-- .github/workflows/mobile-android_build.yml | 417 ++++++---------- .github/workflows/mobile-android_tests.yml | 137 +++--- .github/workflows/mobile-asan.yml | 79 ++-- .github/workflows/mobile-cc_tests.yml | 83 ++-- .../workflows/mobile-compile_time_options.yml | 297 ++++++------ .github/workflows/mobile-core.yml | 83 ++-- .github/workflows/mobile-coverage.yml | 100 ++-- .github/workflows/mobile-docs.yml | 111 +++-- .github/workflows/mobile-format.yml | 236 +++++---- .github/workflows/mobile-ios_build.yml | 446 +++++++----------- .github/workflows/mobile-ios_tests.yml | 131 ++--- .github/workflows/mobile-perf.yml | 191 ++++---- .github/workflows/mobile-release.yml | 10 +- .../workflows/mobile-release_validation.yml | 114 +++-- .github/workflows/mobile-tsan.yml | 82 ++-- .github/workflows/request.yml | 39 ++ .github/workflows/workflow-complete.yml | 63 --- BUILD | 2 +- ci/run_envoy_docker.sh | 3 +- mobile/ci/mac_ci_setup.sh | 5 +- mobile/ci/test_size_regression.sh | 4 +- mobile/tools/what_to_run.sh | 52 -- tools/dependency/BUILD | 8 +- tools/dependency/version.sh | 2 +- 52 files changed, 3212 insertions(+), 2405 deletions(-) delete mode 100644 .github/actions/env/action.yml delete mode 100644 .github/actions/verify/examples/setup/action.yml create mode 100644 .github/config.yml create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/_cache.yml delete mode 100644 .github/workflows/_cache_docker.yml delete mode 100644 .github/workflows/_ci.yml delete mode 100644 .github/workflows/_env.yml create mode 100644 .github/workflows/_finish.yml create mode 100644 .github/workflows/_load.yml create mode 100644 .github/workflows/_mobile_container_ci.yml create mode 100644 .github/workflows/_request.yml create mode 100644 .github/workflows/_run.yml create mode 100644 .github/workflows/_start.yml delete mode 100644 .github/workflows/_workflow-start.yml create mode 100644 .github/workflows/command.yml delete mode 100644 .github/workflows/commands.yml create mode 100644 .github/workflows/request.yml delete mode 100644 .github/workflows/workflow-complete.yml delete mode 100755 mobile/tools/what_to_run.sh diff --git a/.github/actions/env/action.yml b/.github/actions/env/action.yml deleted file mode 100644 index 40b913b1ea62..000000000000 --- a/.github/actions/env/action.yml +++ /dev/null @@ -1,196 +0,0 @@ -inputs: - build_image_tag: - type: string - required: true - build_image_repo: - type: string - required: true - build_image_mobile_sha: - type: string - required: true - build_image_sha: - type: string - required: true - - repo-ref: - type: string - repo-ref-sha: - type: string - repo-ref-name: - type: string - - trusted_bots: - type: string - default: | - trigger-release-envoy[bot] - - check-mobile-run: - type: boolean - default: true - -outputs: - build_image_ubuntu: - value: ${{ steps.build.outputs.build_image_ubuntu }} - build_image_ubuntu_mobile: - value: ${{ steps.build.outputs.build_image_ubuntu_mobile }} - - mobile_android_build: - value: ${{ steps.should_run.outputs.mobile_android_build }} - mobile_android_build_all: - value: ${{ steps.should_run.outputs.mobile_android_build_all }} - mobile_android_tests: - value: ${{ steps.should_run.outputs.mobile_android_tests }} - mobile_asan: - value: ${{ steps.should_run.outputs.mobile_asan }} - mobile_cc_tests: - value: ${{ steps.should_run.outputs.mobile_cc_tests }} - mobile_compile_time_options: - value: ${{ steps.should_run.outputs.mobile_compile_time_options }} - mobile_coverage: - value: ${{ steps.should_run.outputs.mobile_coverage }} - mobile_formatting: - value: ${{ steps.should_run.outputs.mobile_formatting }} - mobile_ios_build: - value: ${{ steps.should_run.outputs.mobile_ios_build }} - mobile_ios_build_all: - value: ${{ steps.should_run.outputs.mobile_ios_build_all }} - mobile_ios_tests: - value: ${{ steps.should_run.outputs.mobile_ios_tests }} - mobile_release_validation: - value: ${{ steps.should_run.outputs.mobile_release_validation }} - mobile_tsan: - value: ${{ steps.should_run.outputs.mobile_tsan }} - repo-ref: - value: ${{ steps.context.outputs.repo-ref }} - repo-ref-name: - value: ${{ steps.context.outputs.repo-ref-name }} - repo-ref-pr-number: - value: ${{ steps.context.outputs.repo-ref-pr-number }} - repo-ref-sha: - value: ${{ steps.context.outputs.repo-ref-sha }} - repo-ref-sha-short: - value: ${{ steps.context.outputs.repo-ref-sha-short }} - repo-ref-title: - value: ${{ steps.context.outputs.repo-ref-title }} - trusted: - value: ${{ steps.trusted.outputs.trusted }} - version-dev: - value: ${{ steps.context.outputs.version-dev }} - version-patch: - value: ${{ steps.context.outputs.version-patch }} - -runs: - using: composite - steps: - # Pull request/targets are _never_ trusted. - # - # For dispatch events, only specified bots are trusted. - # - # Commits to a branch are always trusted. - # - # If code is trusted its not allowed to check out any - # non-ancestor commit of a stable branch. - # - # Untrusted code can check out any commit. - - id: trusted - name: 'Check if its a trusted run' - run: | - TRUSTED=1 - ACTOR="${{ github.actor }}" - if [[ "$ACTOR" =~ \[bot\] ]]; then - TRUSTED_BOT= - TRUSTED_BOTS=(${{ inputs.trusted_bots }}) - for bot in ${TRUSTED_BOTS[@]}; do - if [[ "$bot" == "$ACTOR" ]]; then - # Trusted bot account, ie non-PR - TRUSTED_BOT=1 - break - fi - done - if [[ -z "$TRUSTED_BOT" ]]; then - echo "Not trusted bot account" - TRUSTED= - fi - fi - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "pull_request_target" ]]; then - echo "Not trusted pull_request event" - TRUSTED= - fi - if [[ -n "$TRUSTED" ]]; then - echo "trusted=true" >> "$GITHUB_OUTPUT" - else - echo "trusted=false" >> "$GITHUB_OUTPUT" - fi - shell: bash - - # If we are in a trusted CI run then the provided commit _must_ be either the latest for - # this branch, or an antecdent. - - run: | - if ! git merge-base --is-ancestor "${{ inputs.repo-ref }}" HEAD &> /dev/null; then - echo "Provided Envoy ref (${{ inputs.repo-ref }}) is not an ancestor of current branch" >&2 - exit 1 - fi - git checkout "${{ inputs.repo-ref }}" - if: ${{ steps.trusted.outputs.trusted == 'true' && inputs.repo-ref }} - name: Check provided ref - shell: bash - - - if: ${{ inputs.check-mobile-run != 'false' }} - id: should_run - name: 'Check what to run' - run: ./mobile/tools/what_to_run.sh - shell: bash - - - id: context - name: 'CI context' - run: | - if grep dev VERSION.txt; then - VERSION_DEV="$(cat VERSION.txt | cut -d- -f2)" - else - VERSION_DEV="" - fi - VERSION_PATCH="$(cat VERSION.txt | cut -d- -f1 | rev | cut -d. -f1 | rev)" - # TODO: strip merge from pr names - REF_NAME=${{ inputs.repo-ref-name || github.ref_name }} - if [[ "$REF_NAME" =~ ^refs/pull/ ]]; then - REF_NAME="${REF_NAME:10}" - REF_PR_NUMBER="$(echo "${REF_NAME}" | cut -d/ -f1)" - elif [[ "${{ github.event_name }}" == 'pull_request' ]]; then - REF_PR_NUMBER="$(echo "${REF_NAME}" | cut -d/ -f1)" - fi - echo "SET PR NUMBER: ${REF_PR_NUMBER}" - - REF="${{ steps.trusted.outputs.trusted != 'true' && inputs.repo-ref || '' }}" - REF_SHA=${{ inputs.repo-ref-sha || github.event.pull_request.head.sha || github.sha }} - REF_SHA_SHORT="${REF_SHA:0:7}" - REF_TITLE=( - "${{ steps.trusted.outputs.trusted == 'true' && 'postsubmit' || 'pr' }}/" - "${REF_NAME}" - "@${REF_SHA_SHORT}") - REF_TITLE="$(printf %s "${REF_TITLE[@]}" $'\n')" - { - echo "repo-ref=$REF" - echo "repo-ref-name=$REF_NAME" - echo "repo-ref-pr-number=$REF_PR_NUMBER" - echo "repo-ref-sha=$REF_SHA" - echo "repo-ref-title=$REF_TITLE" - echo "repo-ref-sha-short=$REF_SHA_SHORT" - echo "version-dev=$VERSION_DEV" - echo "version-patch=$VERSION_PATCH" - } >> "$GITHUB_OUTPUT" - shell: bash - - - id: build - name: 'Check current build images' - run: | - { - echo "build_image_ubuntu=${BUILD_IMAGE_UBUNTU_REPO}:${BUILD_IMAGE_UBUNTU}@sha256:${BUILD_IMAGE_UBUNTU_SHA}" - echo "build_image_ubuntu_mobile=${BUILD_IMAGE_UBUNTU_REPO}:mobile-${BUILD_IMAGE_UBUNTU}@sha256:${BUILD_IMAGE_UBUNTU_MOBILE_SHA}" - } >> "$GITHUB_OUTPUT" - env: - # TODO(phlax): derive these from a config file - BUILD_IMAGE_UBUNTU_REPO: ${{ inputs.build_image_repo }} - BUILD_IMAGE_UBUNTU: ${{ inputs.build_image_tag }} - BUILD_IMAGE_UBUNTU_SHA: ${{ inputs.build_image_sha }} - BUILD_IMAGE_UBUNTU_MOBILE_SHA: ${{ inputs.build_image_mobile_sha }} - shell: bash diff --git a/.github/actions/verify/examples/setup/action.yml b/.github/actions/verify/examples/setup/action.yml deleted file mode 100644 index ba7c704c769b..000000000000 --- a/.github/actions/verify/examples/setup/action.yml +++ /dev/null @@ -1,37 +0,0 @@ -inputs: - ref: - type: string - required: true - bucket: - type: string - default: envoy-pr - -runs: - using: composite - steps: - - id: url - run: | - echo "base=https://storage.googleapis.com/${{ inputs.bucket }}/${REF:0:7}/docker" \ - >> "$GITHUB_OUTPUT" - env: - REF: ${{ inputs.ref }} - shell: bash - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.1.7 - with: - url: "${{ steps.url.outputs.base }}/envoy.tar" - variant: dev - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.1.7 - with: - url: "${{ steps.url.outputs.base }}/envoy-contrib.tar" - variant: contrib-dev - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.1.7 - with: - url: "${{ steps.url.outputs.base }}/envoy-google-vrp.tar" - variant: google-vrp-dev - - run: docker images | grep envoy - shell: bash - - run: | - export DEBIAN_FRONTEND=noninteractive - sudo apt-get -qq update -y - sudo apt-get -qq install -y --no-install-recommends expect - shell: bash diff --git a/.github/config.yml b/.github/config.yml new file mode 100644 index 000000000000..5b13aa93cca9 --- /dev/null +++ b/.github/config.yml @@ -0,0 +1,253 @@ +agent-ubuntu: ubuntu-22.04 +build-image: + # Authoritative configuration for build image/s + repo: envoyproxy/envoy-build-ubuntu + sha: 8781bc7e431b754c142edbfc937905fdf343db91f3fe19bbf54c362828db9849 + mobile-sha: 292273a428ebb093cc69c2f225c7c4218ff2ea84e0cb6ee80e7a6771a09e026f + # this is authoritative, but is not currently used in github ci + gcr-sha: af98298b715958527d3c3b42ae968a09223d237c843e129c586b7e76b8c69a0d + tag: 7467652575122d8d54e767a68f141598bd855383 + +config: + envoy: + icon: >- + [![](https://avatars.githubusercontent.com/u/30125649?s=24&v=4)](#) + +checks: + # Checks: this configures which _checks_ will be activated or skipped + # + # The configured _names_ need to match the checks configured for the repo + # + # Any check that is marked as `required` but is not triggered by the run + # config above in a given CI run is marked as `skipped` + # + # For example if macos is marked as `required: true` but then has a path + # selection that means its doesnt run the check will be `skipped` and pass + macos: + name: Envoy/macOS + required: true + on-run: + - build-macos + mobile-android: + name: Mobile/Android + required: true + on-run: + - mobile-android + mobile-android-tests: + name: Mobile/Android tests + required: true + on-run: + - mobile-android-tests + mobile-asan: + name: Mobile/ASAN + required: true + on-run: + - mobile-asan + mobile-cc: + name: Mobile/cc + required: true + on-run: + - mobile-cc + mobile-compile-time-options: + name: Mobile/Compile time options + on-run: + - mobile-compile-time-options + mobile-core: + name: Mobile/Core + required: true + on-run: + - mobile-core + mobile-coverage: + name: Mobile/Coverage + required: true + on-run: + - mobile-coverage + mobile-docs: + name: Mobile/Docs + on-run: + - mobile-docs + mobile-format: + name: Mobile/Format + required: true + on-run: + - mobile-format + mobile-ios: + name: Mobile/ios + required: true + cache: + on-run: + - mobile-ios + mobile-ios-tests: + name: Mobile/iOS tests + required: true + on-run: + - mobile-ios-tests + mobile-perf: + name: Mobile/Perf + required: true + on-run: + - mobile-perf + mobile-release-validation: + name: Mobile/Release validation + required: true + on-run: + - mobile-release-validation + mobile-tsan: + name: Mobile/TSAN + required: true + on-run: + - mobile-tsan + prechecks: + name: Envoy/Prechecks + on-run: + - precheck-deps + required: true + # yamllint disable rule:line-length + advice: + general: | + ### Ensuring your commits are signed off + + You can set up DCO using Envoy's git hooks. + + ### Git hooks + + To set this up, do the following: + + ```console + $ ./support/bootstrap + ``` + + If you only want the DCO check you can do the following to disable the + other hooks + + ```console + $ echo NO_VERIFY=1 > .env + ``` + deps: | + ### Advice on updating dependencies + + General information about Envoy's depdendencies [can be found here](https://github.com/envoyproxy/envoy/blob/main/DEPENDENCY_POLICY.md) + format: | + ### Advice on correct formatting + + Envoy ensures a minimum standard for all files in the repository. + + You are strongly advised to heed the following CI notice: + + ```console + Please fix your editor to ensure: + + - no trailing whitespace + - no preceding mixed tabs/spaces + - all files end with a newline + ``` + # yamllint enable rule:line-length + publish: + name: >- + Envoy/Publish and verify + on-run: + - publish + - verify + required: true + windows: + name: Envoy/Windows + required: true + on-run: + - build-windows + +run: + build-windows: + build-macos: + mobile-android: + mobile-android-all: + paths: + - mobile/**/* + - bazel/repository_locations.bzl + - .bazelrc + - .bazelversion + - .github/workflows/mobile-* + - .github/workflows/_env.yml + - tools/code_format/check_format.py + - bazel/external/quiche.BUILD + mobile-android-tests: + mobile-asan: + mobile-cc: + mobile-compile-time-options: + mobile-coverage: + mobile-core: + mobile-format: + mobile-ios: + mobile-ios-all: + paths: + - mobile/**/* + - bazel/repository_locations.bzl + - .bazelrc + - .bazelversion + - .github/workflows/mobile-* + - .github/workflows/_env.yml + - tools/code_format/check_format.py + - bazel/external/quiche.BUILD + mobile-ios-tests: + mobile-perf: + mobile-release-validation: + mobile-tsan: + precheck-deps: + paths: + - ".github/**/*" + - "**/*.bzl" + - "**/requirements.txt" + publish: + verify: + +tables: + env: + collapse: true + title: Environment + table-title: Request variables + filter: | + .request + | del(.["build-image" as $prefix | keys[] | select(startswith($prefix))]) + | del(.["version" as $prefix | keys[] | select(startswith($prefix))]) + | .actor = "\"\(.actor.name)\" @\(.actor.name)" + build-image: + collapse: true + title: Build image + table-title: Container image/s (as used in this CI run) + filter: | + "https://hub.docker.com/r/envoyproxy/envoy-build-ubuntu/tags?page=1&name=" as $dockerLink + | .request["build-image"] + | del(.changed) + | with_entries( + .value as $v + | ($v | split(":") | .[1] | split("@") | .[0]) as $tag + | .value = "[\($v | split("@") | .[0])](\($dockerLink)\($tag))") + build-image-current: + collapse: true + title: Build image (current) + table-title: Current or previous container image + filter: | + "https://hub.docker.com/r/envoyproxy/envoy-build-ubuntu/tags?page=1&name=" as $dockerLink + | if .request["build-image"].changed then + .request["build-image-current"] + | with_entries( + .value as $v + | ($v | split(":") | .[1] | split("@") | .[0]) as $tag + | .value = "[\($v | split("@") | .[0])](\($dockerLink)\($tag))") + else {} end + version: + collapse: true + title: Version + table-title: Envoy version (as used in this CI run) + filter: | + .request.version + | del(.changed) + version-current: + collapse: true + title: Version (current) + table-title: Current or previous version + filter: | + if .request.version.changed then + .request["version-current"] + else + {} + end diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000000..743c7f39acdd --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,198 @@ +## CI configuration + +CI is configured in .github/config.yml. + +The configuration is per-branch and in this way different branches can have a different +runtime configuration. + +In a pull request only 2 things are read from the config.yml submitted in the request: + +- version +- build image + +As these can change the way the CI runs they are allowed to change. No other configuration +is read from the pull request itself. + +### Checks + +Which checks should run against a commit or PR is configured under the `checks` key. + +The names of these checks should match any checks that are set to required for the repo, +and if a check is required this should be set in the config to ensure the check is marked +as skipped if the related runs are skipped. + +### Runs + +This controls which workflows run, and where necessary which jobs in the workflows. + +This paths can be configured with glob matches to match changed files. + +Paths are always matched for PRs. + +For push requests the config can be set to: + +- always (default): Always runs +- paths: Runs when paths match +- never: Doesnt run on pushes + +## CI requests + +### All CI is requested + +Whether triggered by push event or a pull_request all CI should be viewed as "requested". + +This is very important as it means we can treat incoming triggers in much the same way +as we might handle an incoming web request. + +Much like a web request, CI requests may be "trusted" or "untrusted" and as a consequence +have more or less capability or access. + +Again, much like web requests, CI requests cannot be assumed to be safe. + +Any incoming data - critically data over which a user has the capability to change should +be treated in the same way that user data is handled in a web request. + +Failure to do this opens our CI up to many of the same attacks you might expect in a web scenario +- mostly injection attacks of various sorts. + +### Requests are always made _from_ the triggering branch + +The only CI workflow that is required/used on any branch other than `main` is `request.yml`. + +This file contains any custom configurations required by the branch - for example, build images. + +The request workflow on any branch always delegates to the `_request.yml` on `main`. + +The `_request.yml` workflow contains all required configuration for handling an incoming request. + +All other CI listens for the request workflow to run, and then runs with the requested/parsed data. + +### CI is always run _in_ the context of main + +Other than updating configurations in any given `request.yml` - no CI workflows are parsed +anywhere other than in the context of `main`. + +This means that **all** changes must be made to the `main` workflows for _any_ branch _and_ for PRs. + +Like branch CI, PRs also run in the context of `main` - making changes to these files in a PR will have +no effect until/unless they are landed on the `main` branch. + +### Lifecycle of a CI request + +#### Incoming request: + +Requests can be triggered by a `push` to `main` or a release branch or from a +`pull_request_target` to those branches. + +The `request.yml` file handles this and *must* live on every branch. + +This wf then calls the reusable `_request.yml` workflow, typically on `main`, but +branches can pin this if required. + +#### Request is handled by `_request.yml` workflow: + +This workflow initially reads the `.github/config.yml` from the target branch. + +It uses this to decide which CI and which checks need to be run, and collects information +about the CI request. + +This can be configured on a per-branch basis, by editing the file on the branch. + +This also holds the authoritative build image information. + +Users can request a CI run in a PR with custom build images by editing the config.yml file +on the relevant branch. CI will allow this but flag the change. + +Likewise the version is checked at this stage, and CI flags if it has changed. + +No other CI vars should be editable by users in a PR. + +#### CI check runs *on main* listen for incoming requests and run if required: + +These checks *always* run on `main` but with the repo checked out for the branch or the PR. + +If branches require custom CI this can be added in the relevant file *on main* with +a condition to only trigger for relevant target branch. + +#### Checks are completed at the end of each CI run: + +Currently this reports only on the overall outcome of the CI run and updates the check. + +We can add eg Slack reporting here to notify on failed `main` runs. + +#### Retesting + +PR CI can be retested by issuing `/retest` on the PR. + +This finds the checks related to the latest request and restarts them if they are +failed or cancelled. + +Links on the request page link to the original checks, but the checks themselves will +offer a `reload` button to refresh to the latest version. + +## Branch CI + +All CI is run on `main` - branch CI included. + +The CI will checkout the correct commits and run the CI at that point. + +This means that the CI on `main` should always be able to run the current supported branches. + +There are possible workaround for custom branch CI but the better path is to ensure legacy support +in current `main` or backport any required changes. + +## CI caching + +Currently only x86 Docker images are cached. + +Github has a hard per-repo limit of 10GB cache for CI which is LRU cycled when exceeded. + +This should just be enough to store x86 and arm Docker images for most of our release branches +but will not leave anything to spare. + +We can probably set up a bucket cache for bazel and other caching but this will need to be +done separately for un/trusted CI. + +### Cache mutex + +Due to shortcomings in Github's concurrency algorithm we are using a mutex lock that +is currently stored in the (private) https://github.com/envoyproxy/ci-mutex repository. + +The lock allows CI jobs to wait while the cache is being primed rather than all jobs attempting +to prime the cache simultaneously. + +## Development, testing and CI + +Any Github workflows that use the repository context (`pull_request_target`, `workflow_run`, etc) +**are not tested in Pull Requests** + +This means that changes to CI must be tested/verified in the (private) staging repository. + +### CI enabling vars + +The CI workflows and actions are receptive to certain environment variables being set. + +`ENVOY_CI`: this allows CI to run in non-`envoyproxy/envoy` repos +`ENVOY_MOBILE_CI`: this allows mobile CI to be run in non-`envoyproxy/envoy` repos +`ENVOY_MACOS_CI`: this allows macOS CI to be run in non-`envoyproxy/envoy` repos +`ENVOY_WINDOWS_CI`: this allows Windows CI to be run in non-`envoyproxy/envoy` repos + +With these flags activated the CI runs will respect the normal conditions for running. + +### CI override vars + +The CI workflows will also trigger for specific run settings. + +For example: + +`ENVOY_CI_RUN_MOBILE_ANDROID` would trigger the android CI irrespective of files changed, etc. + +These correspond to the run names as configured in config.yml - for example: + +`ENVOY_CI_RUN_BUILD_MACOS` would ensure the `build-macos` run is triggered. + +### Debugging CI + +Setting `CI_DEBUG` will provide a large amount of runtime information. + +Generally this does not want to be set in a production context. diff --git a/.github/workflows/_cache.yml b/.github/workflows/_cache.yml new file mode 100644 index 000000000000..2191999d6666 --- /dev/null +++ b/.github/workflows/_cache.yml @@ -0,0 +1,69 @@ +name: Cache prime (docker) + +permissions: + contents: read + +on: + workflow_call: + secrets: + app-id: + required: true + app-key: + required: true + inputs: + image-tag: + type: string + required: true + request: + type: string + required: true + lock-repository: + type: string + default: envoyproxy/ci-mutex + +## Docker cache +# +# This workflow will only prime the cache, and should be done separately first, prior +# to any jobs that require it. +# +# For a job that does, you can restore with something like: +# +# steps: +# - uses: envoyproxy/toolshed/gh-actions/docker/cache/restore@actions-v0.0.5 +# with: +# key: "${{ needs.env.outputs.build-image }}" +# + + +jobs: + docker: + runs-on: ubuntu-22.04 + steps: + - uses: envoyproxy/toolshed/gh-actions/appauth@actions-v0.2.8 + id: appauth + name: Appauth (mutex lock) + with: + app_id: ${{ secrets.app-id }} + key: ${{ secrets.app-key }} + - uses: envoyproxy/toolshed/gh-actions/docker/cache/prime@actions-v0.2.8 + id: docker + name: Prime Docker cache (${{ inputs.image-tag }}) + with: + image-tag: ${{ inputs.image-tag }} + lock-token: ${{ steps.appauth.outputs.token }} + lock-repository: ${{ inputs.lock-repository }} + - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.8 + id: data + name: Cache data + with: + input-format: yaml + input: | + cached: ${{ steps.docker.outputs.cached }} + key: ${{ inputs.image-tag }} + - uses: envoyproxy/toolshed/gh-actions/json/table@actions-v0.2.8 + name: Summary + with: + json: ${{ steps.data.outputs.value }} + output-path: GITHUB_STEP_SUMMARY + title: >- + Cache (Docker x64) diff --git a/.github/workflows/_cache_docker.yml b/.github/workflows/_cache_docker.yml deleted file mode 100644 index ab75e52df719..000000000000 --- a/.github/workflows/_cache_docker.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Cache prime (docker) - -permissions: - contents: read - -on: - workflow_call: - inputs: - image_tag: - type: string - required: true - image_repo: - type: string - required: true - image_sha: - type: string - required: true - -concurrency: - group: cache_docker-${{ inputs.image_tag }} - cancel-in-progress: false - -## Docker cache -# -# This workflow will only prime the cache, and should be done separately first, prior -# to any jobs that require it. -# -# For a job that does, you can restore with something like: -# -# steps: -# - uses: envoyproxy/toolshed/gh-actions/docker/cache/restore@actions-v0.0.5 -# with: -# key: "${{ needs.env.outputs.build_image_ubuntu }}" -# - -jobs: - docker: - runs-on: ubuntu-22.04 - steps: - - uses: envoyproxy/toolshed/gh-actions/docker/cache/prime@actions-v0.1.8 - name: Prime Docker cache (${{ inputs.image_repo }}:${{ inputs.image_tag }}@sha256:${{ inputs.image_sha }}) - with: - image_tag: "${{ inputs.image_repo }}:${{ inputs.image_tag }}@sha256:${{ inputs.image_sha }}" diff --git a/.github/workflows/_ci.yml b/.github/workflows/_ci.yml deleted file mode 100644 index fcc79f2f6194..000000000000 --- a/.github/workflows/_ci.yml +++ /dev/null @@ -1,191 +0,0 @@ -name: Envoy CI - -on: - workflow_call: - secrets: - app-id: - app-key: - rbe-key: - inputs: - bazel-extra: - type: string - bazel-rbe-jobs: - type: number - default: 75 - cache-build-image: - type: string - catch-errors: - type: boolean - default: false - command-prefix: - type: string - default: ./ci/run_envoy_docker.sh - command-ci: - type: string - default: ./ci/do_ci.sh - diskspace-hack: - type: boolean - default: false - error-match: - type: string - default: | - ERROR - error: - Error: - notice-match: - type: string - default: | - NOTICE - rbe: - type: boolean - default: true - repo-fetch-depth: - type: number - default: 1 - repo-ref: - type: string - runs-on: - type: string - default: ubuntu-22.04 - skip: - type: boolean - default: false - source: - type: string - steps-pre: - type: string - steps-pre-name: - type: string - steps-post: - type: string - default: | - - run: | - du -ch "%{{ inputs.temp-dir || runner.temp }}" | grep -E "[0-9]{2,}M|[0-9]G" - shell: bash - steps-post-name: - type: string - target: - type: string - required: true - temp-dir: - type: string - trusted: - type: boolean - default: false - upload-name: - type: string - upload-path: - type: string - warning-match: - type: string - default: | - WARNING - warning: - Warning: - -concurrency: - group: | - ${{ github.actor != 'trigger-release-envoy[bot]' - && github.head_ref - || github.run_id - }}-${{ github.workflow }}-${{ inputs.target }} - cancel-in-progress: true - -jobs: - do_ci: - if: ${{ ! inputs.skip }} - runs-on: ${{ inputs.runs-on }} - name: ${{ inputs.command-ci }} ${{ inputs.target }} - steps: - - if: ${{ inputs.cache-build-image }} - name: Restore Docker cache ${{ inputs.cache-build-image && format('({0})', inputs.cache-build-image) || '' }} - uses: envoyproxy/toolshed/gh-actions/docker/cache/restore@actions-v0.1.8 - with: - image_tag: ${{ inputs.cache-build-image }} - - - uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.1.8 - id: checkout - name: Checkout Envoy repository - with: - app_id: ${{ inputs.trusted && secrets.app-id || '' }} - app_key: ${{ inputs.trusted && secrets.app-key || '' }} - config: | - fetch-depth: ${{ ! inputs.trusted && inputs.repo-fetch-depth || 0 }} - # WARNING: This allows untrusted code to run!!! - # If this is set, then anything before or after in the job should be regarded as - # compromised. - ref: ${{ ! inputs.trusted && inputs.repo-ref || github.ref }} - - # If we are in a trusted CI run then the provided commit _must_ be either the latest for - # this branch, or an antecdent. - - run: | - if ! git merge-base --is-ancestor "${{ inputs.repo-ref }}" HEAD; then - echo "Provided Envoy ref (${{ inputs.repo-ref }}) is not an ancestor of current branch" >&2 - exit 1 - fi - git checkout "${{ inputs.repo-ref }}" - if: ${{ inputs.trusted }} - name: Check provided ref - shell: bash - - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - - if: ${{ inputs.diskspace-hack }} - name: Free diskspace - uses: envoyproxy/toolshed/gh-actions/diskspace@actions-v0.1.8 - - run: | - echo "disk space at beginning of build:" - df -h - name: "Check disk space at beginning" - shell: bash - - - uses: envoyproxy/toolshed/gh-actions/using/steps@actions-v0.1.8 - name: Run pre steps - if: ${{ inputs.steps-pre }} - with: - name: ${{ inputs.steps-pre-name }} - steps: ${{ inputs.steps-pre }} - - - uses: envoyproxy/toolshed/gh-actions/github/run@actions-v0.1.8 - name: 'Run CI target ${{ inputs.target }}' - with: - catch-errors: ${{ inputs.catch-errors }} - container-command: ${{ inputs.command-prefix }} - command-prefix: ${{ inputs.command-ci }} - command: ${{ inputs.target }} - source: ${{ inputs.source }} - error-match: ${{ inputs.error-match }} - notice-match: ${{ inputs.notice-match }} - warning-match: ${{ inputs.warning-match }} - env: - GITHUB_TOKEN: ${{ steps.checkout.outputs.token != '' && steps.checkout.outputs.token || secrets.GITHUB_TOKEN }} - ENVOY_DOCKER_BUILD_DIR: ${{ runner.temp }} - ENVOY_RBE: ${{ inputs.rbe != 'false' && 1 || '' }} - RBE_KEY: ${{ secrets.rbe-key }} - BAZEL_BUILD_EXTRA_OPTIONS: >- - --config=remote-ci - ${{ inputs.bazel-extra }} - ${{ inputs.rbe != 'false' && format('--jobs={0}', inputs.bazel-rbe-jobs) || '' }} - BAZEL_FAKE_SCM_REVISION: ${{ github.event_name == 'pull_request' && 'e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9' || '' }} - CI_TARGET_BRANCH: ${{ github.event_name == 'pull_request' && github.event.base.ref || github.ref }} - - - uses: envoyproxy/toolshed/gh-actions/using/steps@actions-v0.1.8 - name: Run post steps - if: ${{ inputs.steps-post }} - with: - name: ${{ inputs.steps-post-name }} - steps: ${{ inputs.steps-post }} - - - run: | - echo "disk space at end of build:" - df -h - name: "Check disk space at end" - shell: bash - - - uses: actions/upload-artifact@v3 - name: Upload artefacts - if: ${{ inputs.upload-name && inputs.upload-path }} - with: - name: ${{ inputs.upload-name }} - path: ${{ inputs.upload-path }} diff --git a/.github/workflows/_env.yml b/.github/workflows/_env.yml deleted file mode 100644 index 8124eac1d940..000000000000 --- a/.github/workflows/_env.yml +++ /dev/null @@ -1,192 +0,0 @@ -name: Environment - -permissions: - contents: read - -on: - workflow_call: - inputs: - # Authoritative configuration for build image/s - build_image_repo: - type: string - default: envoyproxy/envoy-build-ubuntu - build_image_sha: - type: string - default: 8781bc7e431b754c142edbfc937905fdf343db91f3fe19bbf54c362828db9849 - build_image_mobile_sha: - type: string - default: 292273a428ebb093cc69c2f225c7c4218ff2ea84e0cb6ee80e7a6771a09e026f - # this is authoritative, but is not currently used in github ci - build_image_gcr_sha: - type: string - default: af98298b715958527d3c3b42ae968a09223d237c843e129c586b7e76b8c69a0d - build_image_tag: - type: string - default: 7467652575122d8d54e767a68f141598bd855383 - - check-mobile-run: - type: boolean - default: true - prime-build-image: - type: boolean - default: false - repo-ref: - type: string - default: - repo-ref-name: - type: string - default: - repo-ref-sha: - type: string - default: - - outputs: - debug: - value: false - agent_ubuntu: - value: ubuntu-22.04 - build_image_ubuntu: - value: ${{ jobs.repo.outputs.build_image_ubuntu }} - build_image_ubuntu_mobile: - value: ${{ jobs.repo.outputs.build_image_ubuntu_mobile }} - mobile_android_build: - value: ${{ jobs.repo.outputs.mobile_android_build }} - mobile_android_build_all: - value: ${{ jobs.repo.outputs.mobile_android_build_all }} - mobile_android_tests: - value: ${{ jobs.repo.outputs.mobile_android_tests }} - mobile_asan: - value: ${{ jobs.repo.outputs.mobile_asan }} - mobile_cc_tests: - value: ${{ jobs.repo.outputs.mobile_cc_tests }} - mobile_compile_time_options: - value: ${{ jobs.repo.outputs.mobile_compile_time_options }} - mobile_coverage: - value: ${{ jobs.repo.outputs.mobile_coverage }} - mobile_formatting: - value: ${{ jobs.repo.outputs.mobile_formatting }} - mobile_ios_build: - value: ${{ jobs.repo.outputs.mobile_ios_build }} - mobile_ios_build_all: - value: ${{ jobs.repo.outputs.mobile_ios_build_all }} - mobile_ios_tests: - value: ${{ jobs.repo.outputs.mobile_ios_tests }} - mobile_release_validation: - value: ${{ jobs.repo.outputs.mobile_release_validation }} - mobile_tsan: - value: ${{ jobs.repo.outputs.mobile_tsan }} - repo-ref: - value: ${{ jobs.repo.outputs.repo-ref }} - repo-ref-name: - value: ${{ jobs.repo.outputs.repo-ref-name }} - repo-ref-sha: - value: ${{ jobs.repo.outputs.repo-ref-sha }} - repo-ref-sha-short: - value: ${{ jobs.repo.outputs.repo-ref-sha-short }} - repo-ref-title: - value: ${{ jobs.repo.outputs.repo-ref-title }} - trusted: - value: ${{ jobs.repo.outputs.trusted }} - version-dev: - value: ${{ jobs.repo.outputs.version-dev }} - version-patch: - value: ${{ jobs.repo.outputs.version-patch }} - -concurrency: - group: | - ${{ github.actor != 'trigger-release-envoy[bot]' - && github.head_ref - || github.run_id - }}-${{ github.workflow }}-env - cancel-in-progress: true - -jobs: - repo: - if: ${{ github.repository == 'envoyproxy/envoy' || vars.ENVOY_CI }} - runs-on: ubuntu-22.04 - permissions: - contents: read - pull-requests: read - outputs: - build_image_ubuntu: ${{ steps.env.outputs.build_image_ubuntu }} - build_image_ubuntu_mobile: ${{ steps.env.outputs.build_image_ubuntu_mobile }} - mobile_android_build: ${{ steps.env.outputs.mobile_android_build }} - mobile_android_build_all: ${{ steps.env.outputs.mobile_android_build_all }} - mobile_android_tests: ${{ steps.env.outputs.mobile_android_tests }} - mobile_asan: ${{ steps.env.outputs.mobile_asan }} - mobile_cc_tests: ${{ steps.env.outputs.mobile_cc_tests }} - mobile_compile_time_options: ${{ steps.env.outputs.mobile_compile_time_options }} - mobile_coverage: ${{ steps.env.outputs.mobile_coverage }} - mobile_formatting: ${{ steps.env.outputs.mobile_formatting }} - mobile_ios_build: ${{ steps.env.outputs.mobile_ios_build }} - mobile_ios_build_all: ${{ steps.env.outputs.mobile_ios_build_all }} - mobile_ios_tests: ${{ steps.env.outputs.mobile_ios_tests }} - mobile_release_validation: ${{ steps.env.outputs.mobile_release_validation }} - mobile_tsan: ${{ steps.env.outputs.mobile_tsan }} - repo-ref: ${{ steps.ref.outputs.value }} - repo-ref-name: ${{ steps.env.outputs.repo-ref-name }} - repo-ref-sha: ${{ steps.env.outputs.repo-ref-sha }} - repo-ref-sha_short: ${{ steps.env.outputs.repo-ref-sha-short }} - repo-ref-title: ${{ steps.env.outputs.repo-ref-title }} - trusted: ${{ steps.env.outputs.trusted }} - version-dev: ${{ steps.env.outputs.version-dev }} - version-patch: ${{ steps.env.outputs.version-patch }} - steps: - - uses: actions/checkout@v4 - name: Checkout Envoy repository - with: - fetch-depth: ${{ ! (inputs.check-mobile-run || ! startsWith(github.event_name, 'pull_request')) && 1 || 0 }} - # WARNING: This allows untrusted code to run!!! - # If this is set, then anything before or after in the job should be regarded as - # compromised. - ref: ${{ startsWith(github.event_name, 'pull_request') && inputs.repo-ref || '' }} - - - uses: ./.github/actions/env - name: Generate environment variables - id: env - with: - check-mobile-run: ${{ inputs.check-mobile-run }} - repo-ref: ${{ inputs.repo-ref }} - repo-ref-name: ${{ inputs.repo-ref-name }} - repo-ref-sha: ${{ inputs.repo-ref-sha }} - build_image_repo: ${{ inputs.build_image_repo }} - build_image_tag: ${{ inputs.build_image_tag }} - build_image_mobile_sha: ${{ inputs.build_image_mobile_sha }} - build_image_sha: ${{ inputs.build_image_sha }} - - uses: envoyproxy/toolshed/gh-actions/github/merge-commit@actions-v0.1.8 - id: merge-commit - if: ${{ github.event_name == 'pull_request_target' }} - with: - repository: ${{ github.repository }} - pr: ${{ github.event.number }} - token: ${{ secrets.GITHUB_TOKEN }} - - name: 'Set ref' - id: ref - run: | - echo "value=${{ steps.merge-commit.outputs.sha || steps.env.outputs.repo_ref }}" >> $GITHUB_OUTPUT - - - name: 'Print env' - run: | - echo "version-dev=${{ steps.env.outputs.version-dev }}" - echo "version-patch=${{ steps.env.outputs.version-patch }}" - echo "trusted=${{ steps.env.outputs.trusted }}" - echo "repo-ref=${{ steps.ref.outputs.value }}" - echo "repo-ref-name=${{ steps.env.outputs.repo-ref-name }}" - echo "repo-ref-pr-number=${{ steps.env.outputs.repo-ref-pr-number }}" - echo "repo-ref-sha=${{ steps.env.outputs.repo-ref-sha }}" - echo "repo-ref-sha-short=${{ steps.env.outputs.repo-ref-sha-short }}" - echo "repo-ref-title=${{ steps.env.outputs.repo-ref-title }}" - echo "build_image_ubuntu=${{ steps.env.outputs.build_image_ubuntu }}" - echo "build_image_ubuntu_mobile=${{ steps.env.outputs.build_image_ubuntu_mobile }}" - echo - if [[ -n "${{ steps.env.outputs.repo-ref-pr-number }}" ]]; then - echo "PR: https://github.com/${{ github.repository }}/pull/${{ steps.env.outputs.repo-ref-pr-number }}" - fi - - cache: - if: ${{ inputs.prime-build-image }} - uses: ./.github/workflows/_cache_docker.yml - with: - image_repo: ${{ inputs.build_image_repo }} - image_tag: ${{ inputs.build_image_tag }} - image_sha: ${{ inputs.build_image_sha }} diff --git a/.github/workflows/_finish.yml b/.github/workflows/_finish.yml new file mode 100644 index 000000000000..7ce4f17e7e8d --- /dev/null +++ b/.github/workflows/_finish.yml @@ -0,0 +1,118 @@ +name: Workflow/complete + +permissions: + contents: read + + +on: + # Do not run untrusted code here + workflow_call: + secrets: + app-id: + required: true + app-key: + required: true + inputs: + needs: + type: string + required: true + template-check-text: + type: string + default: | + ## \($icon) Check run finished (\($outcome.name) \($outcome.icon)) + + ## The check run can be viewed here: + + # \($icon) \($run_link) + + ### Check started by + + ## \($linked_title) + + \($request.summary.title) + +env: + CI_DEBUG: ${{ vars.CI_DEBUG && true || false }} + + +jobs: + complete: + runs-on: ubuntu-22.04 + permissions: + actions: read + contents: read + steps: + - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.8 + name: Incoming data + id: needs + with: + input: | + repo: ${{ github.repository }} + run_id: ${{ github.run_id }} + outcomes: ${{ toJSON(fromJSON(inputs.needs).*.result) }} + load: ${{ toJSON(fromJSON(inputs.needs).load.outputs) }} + input-format: yaml + print-result: ${{ fromJSON(env.CI_DEBUG || 'false') && true || false }} + filter: | + .repo as $repo + | .run_id as $run_id + | .needs as $result + | .load as $load + | $load["check-id"] as $check_id + | $load["run-id"] as $workflow_id + | (.load.request | fromjson) as $request + | $request.config.envoy.icon as $icon + | $request.summary["linked-title"] as $linked_title + | .outcomes + | if any(. == "failure") then + {name: "failure", icon: ":x:"} + elif any(. == "cancelled") then + {name: "cancelled", icon: ""} + elif all(. == "skipped") then + {name: "skipped", icon: ""} + else + {name: "success", icon: ":heavy_check_mark:"} + end + | . as $outcome + | "\($request.check.name) (\($request.summary.title))" as $run_link_text + | "[\($run_link_text)](https://github.com/\($repo)/actions/runs/\($run_id))" as $run_link + | "${{ inputs.template-check-text }}" as $text + | {"summary-title": "\($icon) \($request.check.name) complete (\($outcome.name))", + "check-id": $check_id, + check: { + name: $request.check.name, + head_sha: $request.request.sha, + status: "completed", + conclusion: $outcome.name, + external_id: "\($run_id)", + output: { + title: "\($request.check.name) (\($outcome.name))", + summary: "Check has finished", + text: $text}}} + + - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.8 + name: Print summary + with: + input: ${{ steps.needs.outputs.value }} + filter: | + "## \(.["summary-title"])" + options: -r + output-path: GITHUB_STEP_SUMMARY + - uses: envoyproxy/toolshed/gh-actions/appauth@actions-v0.2.8 + name: Appauth + id: appauth + with: + app_id: ${{ secrets.app-id }} + key: ${{ secrets.app-key }} + - uses: envoyproxy/toolshed/gh-actions/github/check/update@actions-v0.2.8 + name: Update check + with: + data: ${{ toJSON(fromJSON(steps.needs.outputs.value).check) }} + id: ${{ fromJSON(steps.needs.outputs.value).check-id }} + token: ${{ steps.appauth.outputs.token }} + + # This is necessary to ensure that any retests have their checks updated + - name: Fail the job + if: ${{ fromJSON(steps.needs.outputs.value).check.conclusion != 'success' }} + run: | + exit 1 diff --git a/.github/workflows/_load.yml b/.github/workflows/_load.yml new file mode 100644 index 000000000000..4e1d4070c76e --- /dev/null +++ b/.github/workflows/_load.yml @@ -0,0 +1,153 @@ +name: Request/load + +permissions: + contents: read + +on: + workflow_call: + secrets: + app-id: + required: true + app-key: + required: true + lock-app-id: + required: true + lock-app-key: + required: true + + inputs: + agent-ubuntu: + type: string + default: ubuntu-22.04 + cache-docker: + type: boolean + default: true + check-name: + type: string + required: true + check-title: + type: string + default: + head-sha: + type: string + default: + run-id: + type: string + default: ${{ github.event.workflow_run.id }} + template-request-summary: + type: string + default: | + ## \($linkedTitle) + + \($summary) + + \($extra) + + outputs: + build-image: + value: ${{ jobs.request.outputs.build-image }} + build-image-mobile: + value: ${{ jobs.request.outputs.build-image-mobile }} + check-id: + value: ${{ jobs.request.outputs.check-id }} + check-name: + value: ${{ inputs.check-name }} + request: + value: ${{ jobs.request.outputs.request }} + run-id: + value: ${{ inputs.run-id }} + trusted: + value: ${{ jobs.request.outputs.trusted }} + +concurrency: + group: | + ${{ github.actor != 'trigger-release-envoy[bot]' + && github.head_ref + || github.run_id + }}-${{ github.workflow }}-env + cancel-in-progress: true + +env: + CI_DEBUG: ${{ vars.CI_DEBUG && true || false }} + + +jobs: + request: + if: ${{ github.repository == 'envoyproxy/envoy' || vars.ENVOY_CI }} + runs-on: ubuntu-22.04 + permissions: + actions: read + contents: read + pull-requests: read + outputs: + build-image: ${{ toJSON(fromJSON(steps.request-output.outputs.value).request.build-image) }} + build-image-mobile: ${{ fromJSON(steps.request-output.outputs.value).request.build-image-mobile }} + check-id: ${{ fromJSON(steps.request-output.outputs.value).check.check-id }} + request: ${{ steps.request-output.outputs.value }} + trusted: ${{ fromJSON(steps.request-output.outputs.value).request.trusted }} + skip: ${{ fromJSON(steps.request-output.outputs.value).check.action != 'RUN' }} + steps: + # Load env data + # Handle any failure in triggering job + # Remove the `checks` object, and shift the one we care about to `check` + # Prepare a check request + - uses: envoyproxy/toolshed/gh-actions/github/env/load@actions-v0.2.8 + name: Load env + id: data + with: + run-id: ${{ inputs.run-id }} + check-name: ${{ inputs.check-name }} + head-sha: ${{ inputs.head-sha }} + env: + GH_TOKEN: ${{ github.token }} + + - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.8 + name: Print request summary + with: + input: ${{ steps.data.outputs.data }} + output-path: GITHUB_STEP_SUMMARY + options: -r + filter: | + .data as $data + | if ($data.check.action != "RUN") then + "### ${{ github.workflow }} was skipped" + else "" end + | . as $extra + | $data.summary.summary as $summary + | $data.summary["linked-title"] as $linkedTitle + | "${{ inputs.template-request-summary }}" + + # Update the check + - uses: envoyproxy/toolshed/gh-actions/appauth@actions-v0.2.8 + name: Appauth + id: appauth + with: + app_id: ${{ secrets.app-id }} + key: ${{ secrets.app-key }} + - uses: envoyproxy/toolshed/gh-actions/github/check/update@actions-v0.2.8 + name: Update check + if: ${{ fromJSON(steps.data.outputs.data).data.check.check-id }} + with: + data: ${{ toJSON(fromJSON(steps.data.outputs.data).check) }} + id: ${{ fromJSON(steps.data.outputs.data).data.check.check-id }} + token: ${{ steps.appauth.outputs.token }} + + - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.8 + id: request-output + name: Load request + with: + input: ${{ steps.data.outputs.data }} + filter: | + .data + print-result: ${{ fromJSON(env.CI_DEBUG || 'false') && true || false }} + + cache: + secrets: + app-id: ${{ secrets.lock-app-id }} + app-key: ${{ secrets.lock-app-key }} + uses: ./.github/workflows/_cache.yml + needs: request + if: ${{ inputs.cache-docker && ! fromJSON(needs.request.outputs.skip) }} + with: + request: ${{ toJSON(needs.request.outputs) }} + image-tag: ${{ fromJSON(needs.request.outputs.build-image).default }} diff --git a/.github/workflows/_mobile_container_ci.yml b/.github/workflows/_mobile_container_ci.yml new file mode 100644 index 000000000000..c2d6f79d682a --- /dev/null +++ b/.github/workflows/_mobile_container_ci.yml @@ -0,0 +1,147 @@ +name: Mobile CI + +permissions: + contents: read + +on: + workflow_call: + secrets: + app-id: + app-key: + rbe-key: + ssh-key-extra: + inputs: + args: + type: string + catch-errors: + type: boolean + default: false + checkout-extra: + type: string + default: + command: + type: string + default: ./bazelw + container: + type: string + container-output: + type: string + default: + container-command: + type: string + default: >- + docker run + --volume=${PWD}:/source + --volume=${TMP_ENTRYPOINT}:/tmp/mobile-entrypoint.sh + --volume=/tmp/cache:/root/.cache + --workdir=/source/mobile + --entrypoint=/tmp/mobile-entrypoint.sh + -e GITHUB_TOKEN + -e CC + -e CXX + -e COVERAGE_THRESHOLD + -e BAZEL_BUILD_OPTION_LIST + -e MOBILE_DOCS_CHECKOUT_DIR + diskspace-hack: + type: boolean + default: false + downloads: + type: string + default: + entrypoint: + type: string + default: + entrypoint-DEFAULT: + type: string + default: | + #!/bin/bash -e + export PATH=/opt/llvm/bin:$PATH + exec "$@" + error-match: + type: string + default: | + ERROR + error: + Error: + notice-match: + type: string + default: | + NOTICE + Streaming build results + rbe: + type: boolean + default: true + ref: + type: string + request: + type: string + required: true + runs-on: + type: string + skip: + type: boolean + default: false + source: + type: string + default: + steps-pre: + type: string + steps-pre-name: + type: string + steps-post: + type: string + default: + steps-post-name: + type: string + target: + type: string + required: true + temp-dir: + type: string + timeout-minutes: + type: number + trusted: + type: boolean + default: false + upload-name: + type: string + upload-path: + type: string + warning-match: + type: string + default: | + WARNING + warning: + Warning: + + +jobs: + ci: + uses: ./.github/workflows/_run.yml + name: ${{ inputs.target }} + permissions: + contents: read + packages: read + secrets: + ssh-key-extra: ${{ secrets.ssh-key-extra }} + with: + args: ${{ inputs.args }} + rbe: ${{ inputs.rbe }} + # This always just caches the main build image, the mobile one is layered on top + cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }} + catch-errors: ${{ inputs.catch-errors }} + container-command: ${{ inputs.container-command }} ${{ inputs.container || fromJSON(inputs.request).request.build-image.default }} + container-output: ${{ inputs.container-output }} + command: ${{ inputs.command }} + entrypoint: ${{ inputs.entrypoint || inputs.entrypoint-DEFAULT }} + downloads: ${{ inputs.downloads }} + error-match: ${{ inputs.error-match }} + notice-match: ${{ inputs.notice-match }} + request: ${{ inputs.request }} + source: ${{ inputs.source }} + steps-post: ${{ inputs.steps-post }} + target: ${{ inputs.target }} + trusted: ${{ fromJSON(inputs.request).request.trusted }} + upload-name: ${{ inputs.upload-name }} + upload-path: ${{ inputs.upload-path }} + warning-match: ${{ inputs.warning-match }} diff --git a/.github/workflows/_precheck_deps.yml b/.github/workflows/_precheck_deps.yml index c0578a4e1af5..e0e171533681 100644 --- a/.github/workflows/_precheck_deps.yml +++ b/.github/workflows/_precheck_deps.yml @@ -1,4 +1,4 @@ -name: Publish +name: Precheck/deps permissions: contents: read @@ -6,41 +6,44 @@ permissions: on: workflow_call: inputs: - build-image-ubuntu: - type: string - default: '' dependency-review: type: boolean default: false - repo-ref: + request: type: string + required: true + trusted: + type: boolean + required: true concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }}-publish cancel-in-progress: true + jobs: prechecks: - strategy: - matrix: - include: - - target: deps - rbe: false - uses: ./.github/workflows/_ci.yml - name: ${{ matrix.target }} permissions: contents: read packages: read + uses: ./.github/workflows/_run.yml + name: ${{ matrix.target }} with: - target: ${{ matrix.target }} - rbe: ${{ matrix.rbe }} bazel-extra: '--config=rbe-envoy-engflow' - cache-build-image: ${{ inputs.build-image-ubuntu }} - repo-ref: ${{ inputs.repo-ref }} - catch-errors: true + cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }} + request: ${{ inputs.request }} error-match: | ERROR - ClientConnectorError + error: + Error: + rbe: ${{ matrix.rbe }} + target: ${{ matrix.target }} + trusted: ${{ inputs.trusted }} + strategy: + matrix: + include: + - target: deps + rbe: false dependency-review: runs-on: ubuntu-22.04 @@ -49,7 +52,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 with: - ref: ${{ inputs.repo-ref }} + ref: ${{ fromJSON(inputs.request).request.sha }} persist-credentials: false - name: Dependency Review - uses: actions/dependency-review-action@6c5ccdad469c9f8a2996bfecaec55a631a347034 + uses: actions/dependency-review-action@782951d0fdb2f0e1d2e6045c19db15975387e880 diff --git a/.github/workflows/_request.yml b/.github/workflows/_request.yml new file mode 100644 index 000000000000..e304e2b516fa --- /dev/null +++ b/.github/workflows/_request.yml @@ -0,0 +1,121 @@ +name: Request/incoming + +permissions: + contents: read + +on: + workflow_call: + secrets: + app-id: + required: true + app-key: + required: true + + # Defaults are set .github/config.yml on the `main` branch. + inputs: + config-file: + type: string + default: ./.github/config.yml + +concurrency: + group: | + ${{ github.actor != 'trigger-release-envoy[bot]' + && github.head_ref + || github.run_id + }}-${{ github.workflow }}-env-prime + cancel-in-progress: true + +env: + CI_DEBUG: ${{ (vars.CI_DEBUG || vars.RUNNER_DEBUG) && true || false }} + + +jobs: + incoming: + if: ${{ github.repository == 'envoyproxy/envoy' || vars.ENVOY_CI }} + runs-on: ubuntu-22.04 + permissions: + contents: read + pull-requests: read + outputs: + env: ${{ steps.data.outputs.value }} + config: ${{ steps.config.outputs.config }} + steps: + - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.8 + id: started + name: Create timestamp + with: + options: -r + filter: | + now + - uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.2.8 + id: checkout + name: Checkout Envoy repository + with: + pr: ${{ github.event.number }} + branch: ${{ github.ref_name }} + config: | + fetch-depth: ${{ startsWith(github.event_name, 'pull_request') && 1 || 2 }} + # This step *LOOKS AT* the repo at the point requested + # Its essential that this _job_ *MUST NOT EXECUTE ANY CODE FROM THE CHECKED OUT REPO* + # *ALL* variables collected should be treated as untrusted and should be sanitized before + # use + - name: Generate environment variables from commit + uses: envoyproxy/toolshed/gh-actions/envoy/ci/request@actions-v0.2.8 + id: env + with: + branch-name: ${{ steps.checkout.outputs.branch-name }} + config-file: ${{ inputs.config-file }} + merge-commit: ${{ steps.checkout.outputs.merge-commit }} + started: ${{ steps.started.outputs.value }} + token: ${{ secrets.GITHUB_TOKEN }} + vars: ${{ toJSON(vars) }} + - name: Request summary + id: summary + uses: envoyproxy/toolshed/gh-actions/github/env/summary@actions-v0.2.8 + with: + actor: ${{ toJSON(fromJSON(steps.env.outputs.data).request.actor) }} + base-sha: ${{ fromJSON(steps.env.outputs.data).request.base-sha }} + link: ${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }} + output-path: GITHUB_STEP_SUMMARY + pr: ${{ github.event.number }} + data: ${{ steps.env.outputs.data }} + tables: ${{ toJSON(fromJSON(steps.env.outputs.data).config.tables) }} + icon: ${{ fromJSON(steps.env.outputs.data).config.envoy.icon }} + message: ${{ fromJSON(steps.env.outputs.data).request.message }} + ref: ${{ fromJSON(steps.env.outputs.data).request.ref }} + sha: ${{ fromJSON(steps.env.outputs.data).request.sha }} + target-branch: ${{ fromJSON(steps.env.outputs.data).request.target-branch }} + + - name: Environment data + uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.8 + id: data + with: + input: | + env: ${{ steps.env.outputs.data }} + title: ${{ steps.summary.outputs.title }} + link: ${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }} + summary: ${{ steps.summary.outputs.summary }} + input-format: yaml + filter: | + .title as $title + | .env.config.envoy.icon as $icon + | .link as $link + | "\($icon) Request ([\($title)](\($link)))" as $linkedTitle + | .summary as $summary + | .env + | .summary = { + $summary, + $title, + $link, + "linked-title": $linkedTitle} + | del(.config.tables) + + checks: + if: ${{ github.repository == 'envoyproxy/envoy' || vars.ENVOY_CI }} + needs: incoming + uses: ./.github/workflows/_start.yml + secrets: + app-id: ${{ secrets.app-id }} + app-key: ${{ secrets.app-key }} + with: + env: ${{ needs.incoming.outputs.env }} diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml new file mode 100644 index 000000000000..f42f146c8e6c --- /dev/null +++ b/.github/workflows/_run.yml @@ -0,0 +1,261 @@ +name: Envoy CI + +permissions: + contents: read + +on: + workflow_call: + secrets: + app-id: + app-key: + rbe-key: + ssh-key: + ssh-key-extra: + inputs: + args: + type: string + bazel-extra: + type: string + bazel-rbe-jobs: + type: number + default: 75 + cache-build-image: + type: string + catch-errors: + type: boolean + default: false + checkout-extra: + type: string + container-command: + type: string + default: ./ci/run_envoy_docker.sh + container-output: + type: string + default: + command: + type: string + default: ./ci/do_ci.sh + diskspace-hack: + type: boolean + default: false + diskspace-hack-paths: + type: string + default: + downloads: + type: string + entrypoint: + type: string + default: + error-match: + type: string + default: | + ERROR + error: + Error: + notice-match: + type: string + default: | + NOTICE + Streaming build results + rbe: + type: boolean + default: true + repo-fetch-depth: + type: number + default: 1 + report-pre: + type: string + default: | + - run: | + echo "disk space at beginning of build:" + df -h + shell: bash + report-post: + type: string + default: | + - run: | + echo "disk space at end of build:" + df -h + shell: bash + request: + type: string + required: true + runs-on: + type: string + default: + skip: + type: boolean + default: false + source: + type: string + summary-post: + type: string + default: | + - uses: envoyproxy/toolshed/gh-actions/envoy/run/summary@actions-v0.2.8 + with: + context: %{{ inputs.context }} + steps-pre: + type: string + steps-pre-name: + type: string + steps-post: + type: string + default: | + - run: | + du -ch "%{{ inputs.temp-dir || runner.temp }}" | grep -E "[0-9]{2,}M|[0-9]G" || : + shell: bash + steps-post-name: + type: string + target: + type: string + required: true + temp-dir: + type: string + timeout-minutes: + type: number + trusted: + type: boolean + required: true + upload-name: + type: string + upload-path: + type: string + warning-match: + type: string + default: | + WARNING + warning: + Warning: + working-directory: + type: string + default: . + +concurrency: + group: >- + ${{ github.actor != 'trigger-release-envoy[bot]' + && github.head_ref + || github.run_id + }}-${{ github.workflow }}-${{ inputs.target }} + cancel-in-progress: true + +env: + CI_DEBUG: ${{ vars.CI_DEBUG }} + + +jobs: + ci: + permissions: + contents: read + packages: read + if: ${{ ! inputs.skip }} + runs-on: ${{ inputs.runs-on || fromJSON(inputs.request).config.ci.agent-ubuntu }} + name: ${{ inputs.command }} ${{ inputs.target }} + timeout-minutes: ${{ inputs.timeout-minutes }} + steps: + - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.8 + id: started + name: Create timestamp + with: + options: -r + filter: | + now + # This controls which input vars are exposed to the run action (and related steps) + - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.8 + name: Context + id: context + with: + print-result: ${{ fromJSON(env.CI_DEBUG || 'false') && true || false }} + input: ${{ inputs.request }} + filter: | + . + | .check as $check + | .config as $config + | if "${{ inputs.runs-on }}" != "" then + "${{ inputs.runs-on }}" + else .config.ci["agent-ubuntu"] end + | . as $runsOn + | {"target": "${{ inputs.target }}", + "catch-errors": ${{ inputs.catch-errors }}, + "runs-on": $runsOn, + "job-started": ${{ steps.started.outputs.value }}} + | . * {$config, $check} + - if: ${{ inputs.cache-build-image }} + name: Restore Docker cache ${{ inputs.cache-build-image && format('({0})', inputs.cache-build-image) || '' }} + uses: envoyproxy/toolshed/gh-actions/docker/cache/restore@actions-v0.2.8 + with: + image_tag: ${{ inputs.cache-build-image }} + + - uses: envoyproxy/toolshed/gh-actions/appauth@actions-v0.2.8 + id: appauth + name: Appauth + if: ${{ inputs.trusted }} + with: + app_id: ${{ secrets.app-id }} + key: ${{ secrets.app-key }} + # You cant use a secret as a condition so this always runs even if the app id/key are empty + # - the workaround is to allow the token to be passed through. + token: ${{ github.token }} + token-ok: true + - uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.2.8 + id: checkout + name: Checkout Envoy repository + with: + branch: ${{ fromJSON(inputs.request).request.target-branch }} + config: | + fetch-depth: ${{ inputs.repo-fetch-depth }} + # WARNING: This allows untrusted code to run!!! + # If this is set to run untrusted code, then anything before or after in the job should be regarded as + # compromisable. + ref: ${{ inputs.trusted && fromJSON(inputs.request).request.sha || fromJSON(inputs.request).request.ref }} + fetch-merge-commit: false + pr: ${{ fromJSON(inputs.request).request.pr }} + ssh-key: ${{ inputs.trusted && inputs.ssh-key || '' }} + token: ${{ inputs.trusted && steps.appauth.outputs.token || github.token }} + + # This is currently only use by mobile-docs and can be removed once they are updated to the newer website + - uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.2.8 + id: checkout-extra + name: Checkout extra repository (for publishing) + if: ${{ inputs.checkout-extra }} + with: + config: ${{ inputs.checkout-extra }} + ssh-key: ${{ inputs.trusted && inputs.ssh-key-extra || '' }} + + - uses: envoyproxy/toolshed/gh-actions/github/run@actions-v0.2.8 + name: Run CI ${{ inputs.command }} ${{ inputs.target }} + with: + args: ${{ inputs.args != '--' && inputs.args || inputs.target }} + catch-errors: ${{ inputs.catch-errors }} + command: ${{ inputs.command }} + container-command: ${{ env.CONTAINER_COMMAND || inputs.container-command }} + container-output: ${{ inputs.container-output }} + context: ${{ steps.context.outputs.value }} + diskspace-hack: ${{ inputs.diskspace-hack }} + diskspace-hack-paths: ${{ inputs.diskspace-hack-paths }} + downloads: ${{ inputs.downloads }} + entrypoint: ${{ inputs.entrypoint }} + error-match: ${{ inputs.error-match }} + notice-match: ${{ inputs.notice-match }} + report-pre: ${{ inputs.report-pre }} + report-post: ${{ inputs.report-post }} + source: ${{ inputs.source }} + steps-pre: ${{ inputs.steps-pre }} + steps-pre-name: ${{ inputs.steps-pre-name }} + steps-post: ${{ inputs.steps-post }} + steps-post-name: ${{ inputs.steps-post-name }} + summary-post: ${{ inputs.summary-post }} + upload-name: ${{ inputs.upload-name }} + upload-path: ${{ inputs.upload-path }} + warning-match: ${{ inputs.warning-match }} + working-directory: ${{ inputs.working-directory }} + env: + GITHUB_TOKEN: ${{ steps.checkout.outputs.token && steps.checkout.outputs.token || secrets.GITHUB_TOKEN }} + ENVOY_DOCKER_BUILD_DIR: ${{ runner.temp }} + ENVOY_RBE: ${{ inputs.rbe != 'false' && 1 || '' }} + RBE_KEY: ${{ secrets.rbe-key }} + BAZEL_BUILD_EXTRA_OPTIONS: >- + --config=remote-ci + ${{ inputs.bazel-extra }} + ${{ inputs.rbe != 'false' && format('--jobs={0}', inputs.bazel-rbe-jobs) || '' }} + BAZEL_FAKE_SCM_REVISION: ${{ github.event_name == 'pull_request' && 'e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9' || '' }} + CI_TARGET_BRANCH: ${{ fromJSON(inputs.request).request.target-branch }} diff --git a/.github/workflows/_stage_publish.yml b/.github/workflows/_stage_publish.yml index 8f99a5181dcd..2cbad7b6db32 100644 --- a/.github/workflows/_stage_publish.yml +++ b/.github/workflows/_stage_publish.yml @@ -11,107 +11,79 @@ permissions: on: workflow_call: - inputs: - trusted: - type: boolean - default: false - build_image_ubuntu: - type: string - default: '' - version-dev: - type: string - default: '' - head_ref: - type: string - default: '' - repo-ref: - type: string - sha: - type: string secrets: ENVOY_CI_SYNC_APP_ID: ENVOY_CI_SYNC_APP_KEY: ENVOY_CI_PUBLISH_APP_ID: ENVOY_CI_PUBLISH_APP_KEY: + inputs: + request: + type: string + required: true + trusted: + type: boolean + required: true concurrency: - group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }}-publish + group: >- + ${{ github.actor != 'trigger-release-envoy[bot]' + && github.event.inputs.head_ref + || github.run_id + }}-${{ github.event.workflow.id }}-publish cancel-in-progress: true + jobs: - publish_ci: - if: ${{ ! inputs.trusted }} + publish: + secrets: + app-id: ${{ inputs.trusted && secrets.ENVOY_CI_PUBLISH_APP_ID || '' }} + app-key: ${{ inputs.trusted && secrets.ENVOY_CI_PUBLISH_APP_KEY || '' }} + permissions: + contents: read + packages: read name: ${{ matrix.name || matrix.target }} - uses: ./.github/workflows/_ci.yml + uses: ./.github/workflows/_run.yml with: target: ${{ matrix.target }} rbe: false - cache-build-image: ${{ inputs.build_image_ubuntu }} + cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }} source: ${{ matrix.source }} - trusted: false - repo-ref: ${{ inputs.repo-ref }} + request: ${{ inputs.request }} steps-pre: ${{ matrix.steps-pre }} + trusted: ${{ inputs.trusted }} strategy: fail-fast: false matrix: include: - target: publish name: github - steps-pre: | - - id: short_name - uses: envoyproxy/toolshed/gh-actions/str/sub@actions-v0.1.7 - with: - length: 7 - string: ${{ inputs.repo-ref }} - - uses: envoyproxy/toolshed/gh-actions/fetch@actions-v0.1.7 - with: - url: https://storage.googleapis.com/envoy-pr/%{{ steps.short_name.outputs.string }}/release/release.signed.tar.zst - path: %{{ runner.temp }}/release.signed source: | - export ENVOY_PUBLISH_DRY_RUN=1 - export ENVOY_COMMIT=${{ inputs.sha }} + export ENVOY_COMMIT=${{ fromJSON(inputs.request).request.sha }} export ENVOY_REPO=${{ github.repository }} - - publish: - if: ${{ inputs.trusted }} - name: ${{ matrix.name || matrix.target }} - uses: ./.github/workflows/_ci.yml - with: - target: ${{ matrix.target }} - rbe: false - cache-build-image: ${{ inputs.build_image_ubuntu }} - source: ${{ matrix.source }} - trusted: true - repo-ref: ${{ inputs.repo-ref }} - steps-pre: ${{ matrix.steps-pre }} - secrets: - app-id: ${{ secrets.ENVOY_CI_PUBLISH_APP_ID }} - app-key: ${{ secrets.ENVOY_CI_PUBLISH_APP_KEY }} - permissions: - contents: read - packages: read - strategy: - fail-fast: false - matrix: - include: - - target: publish - name: github + export ENVOY_PUBLISH_DRY_RUN=${{ (fromJSON(inputs.request).request.version.dev || ! inputs.trusted) && 1 || '' }} steps-pre: | - - id: short_name - uses: envoyproxy/toolshed/gh-actions/str/sub@actions-v0.1.7 + - id: url + uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.8 with: - length: 7 - string: ${{ inputs.repo-ref }} - - uses: envoyproxy/toolshed/gh-actions/fetch@actions-v0.1.7 + options: -Rr + input: >- + ${{ inputs.trusted + && fromJSON(inputs.request).request.sha + || fromJSON(inputs.request).request.ref }} + filter: | + .[:7] as $sha + | "release/release.signed.tar.zst" as $path + | if ${{ inputs.trusted }} then + "envoy-postsubmit" + else + "envoy-pr" + end + | . as $bucket + | "https://storage.googleapis.com/\($bucket)/\($sha)/\($path)" + - uses: envoyproxy/toolshed/gh-actions/fetch@actions-v0.2.8 with: - url: https://storage.googleapis.com/envoy-postsubmit/%{{ steps.short_name.outputs.string }}/release/release.signed.tar.zst + url: %{{ steps.url.outputs.value }} path: %{{ runner.temp }}/release.signed - source: | - export ENVOY_COMMIT=${{ inputs.sha }} - export ENVOY_REPO=${{ github.repository }} - if [[ '${{ inputs.version-dev }}' == 'dev' ]]; then - export ENVOY_PUBLISH_DRY_RUN=1 - fi publish_docs: # For normal commits to Envoy main this will trigger an update in the website repo, @@ -126,12 +98,16 @@ jobs: needs: - publish steps: - - uses: envoyproxy/toolshed/gh-actions/dispatch@actions-v0.1.8 + - uses: envoyproxy/toolshed/gh-actions/appauth@actions-v0.2.8 + id: appauth with: app_id: ${{ secrets.ENVOY_CI_SYNC_APP_ID }} - key: "${{ secrets.ENVOY_CI_SYNC_APP_KEY }}" + key: ${{ secrets.ENVOY_CI_SYNC_APP_KEY }} + - uses: envoyproxy/toolshed/gh-actions/dispatch@actions-v0.2.8 + with: ref: main - repository: ${{ inputs.version-dev == 'dev' && 'envoyproxy/envoy-website' || 'envoyproxy/archive' }} + repository: ${{ fromJSON(inputs.request).request.version.dev && 'envoyproxy/envoy-website' || 'envoyproxy/archive' }} + token: ${{ steps.appauth.outputs.token }} workflow: envoy-sync.yaml inputs: | - commit_sha: ${{ inputs.version-dev == 'dev' && github.sha || '' }} + commit_sha: ${{ fromJSON(inputs.request).request.version.dev && github.sha || '' }} diff --git a/.github/workflows/_stage_verify.yml b/.github/workflows/_stage_verify.yml index 79879e0172ab..12c8a43436f0 100644 --- a/.github/workflows/_stage_verify.yml +++ b/.github/workflows/_stage_verify.yml @@ -6,32 +6,39 @@ permissions: on: workflow_call: inputs: + request: + type: string + required: true trusted: type: boolean - default: false - repo-ref: - type: string - given_ref: - type: string + required: true concurrency: - group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }}-verify + group: >- + ${{ github.actor != 'trigger-release-envoy[bot]' + && github.event.inputs.head_ref + || github.run_id + }}-${{ github.event.workflow.id }}-verify cancel-in-progress: true + jobs: verify: + permissions: + contents: read + packages: read name: ${{ matrix.name || matrix.target }} - uses: ./.github/workflows/_ci.yml + uses: ./.github/workflows/_run.yml with: - target: ${{ matrix.target }} - rbe: ${{ matrix.rbe }} cache-build-image: - command-prefix: - source: ${{ matrix.source }} - trusted: ${{ inputs.trusted }} - repo-ref: ${{ inputs.repo-ref }} + container-command: + rbe: ${{ matrix.rbe }} + request: ${{ inputs.request }} runs-on: envoy-x64-small steps-pre: ${{ matrix.steps-pre }} + source: ${{ matrix.source }} + target: ${{ matrix.target }} + trusted: ${{ inputs.trusted }} strategy: fail-fast: false matrix: @@ -42,33 +49,39 @@ jobs: export NO_BUILD_SETUP=1 rbe: false steps-pre: | - - id: short_name - uses: envoyproxy/toolshed/gh-actions/str/sub@actions-v0.1.7 + - id: url + uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.8 with: - length: 7 - string: ${{ inputs.repo-ref }} - - id: gcp - run: | - PREFIX=https://storage.googleapis.com/envoy- - BUCKET=${{ inputs.trusted && 'postsubmit' || 'pr' }} - NAME=%{{ steps.short_name.outputs.string }} - echo "url=${PREFIX}${BUCKET}/${NAME}" >> $GITHUB_OUTPUT - shell: bash - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.1.7 + options: -Rr + input: >- + ${{ inputs.trusted + && fromJSON(inputs.request).request.sha + || fromJSON(inputs.request).request.ref }} + filter: | + .[:7] as $sha + | if ${{ inputs.trusted }} then + "envoy-postsubmit" + else + "envoy-pr" + end + | . as $bucket + | "https://storage.googleapis.com/\($bucket)/\($sha)" + - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.8 with: - url: %{{ steps.gcp.outputs.url }}/docker/envoy.tar + url: %{{ steps.url.outputs.value }}/docker/envoy.tar variant: dev - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.1.7 + - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.8 with: - url: %{{ steps.gcp.outputs.url }}/docker/envoy-contrib.tar + url: %{{ steps.url.outputs.value }}/docker/envoy-contrib.tar variant: contrib-dev - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.1.7 + - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.8 with: - url: %{{ steps.gcp.outputs.url }}/docker/envoy-google-vrp.tar + url: %{{ steps.url.outputs.value }}/docker/envoy-google-vrp.tar variant: google-vrp-dev - run: docker images | grep envoy shell: bash - run: | + # Install expected host packages export DEBIAN_FRONTEND=noninteractive sudo apt-get -qq update -y sudo apt-get -qq install -y --no-install-recommends expect diff --git a/.github/workflows/_start.yml b/.github/workflows/_start.yml new file mode 100644 index 000000000000..67ff2aa805bb --- /dev/null +++ b/.github/workflows/_start.yml @@ -0,0 +1,131 @@ +name: Workflow start +# This workflow is only required for externally triggered jobs that need to manually +# set the check status for a commit/PR + +permissions: + contents: read + +on: + workflow_call: + secrets: + app-id: + required: true + app-key: + required: true + inputs: + details-url: + type: string + default: >- + https://github.com/envoyproxy/envoy/tree/main/.github/workflows + env: + type: string + required: true + run-summary: + type: string + default: >- + The check will start once any required jobs have completed and a VM becomes available + run-title: + type: string + default: >- + Waiting for check ... + skipped-summary: + type: string + default: >- + This check was not triggered in this CI run + skipped-title: + type: string + default: >- + Check was skipped + template-run-text: + type: string + default: | + ## \($icon) Check run pending + + ## Details of the check run will be provided here once it has started. + + ### Check started by + + +env: + CI_DEBUG: ${{ (vars.CI_DEBUG || vars.RUNNER_DEBUG) && true || false }} + + +jobs: + start: + runs-on: ubuntu-22.04 + steps: + - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.8 + id: check-config + name: Prepare check data + with: + print-result: ${{ fromJSON(env.CI_DEBUG || 'false') && true || false }} + input: ${{ inputs.env }} + filter: | + . as $env + | .config.envoy.icon as $icon + | {} + | .["head_sha"] = $env.request.sha + | .details_url = "${{ inputs.details-url }}" + | {run: ., skipped: ., request: $env.summary.summary} + | .run.output.title = "${{ inputs.run-title }}" + | .run.output.summary = "${{ inputs.run-summary }}" + | .run.output.text = "${{ inputs.template-run-text }}" + | .run.status = "queued" + | .skipped.status = "completed" + | .skipped.conclusion = "skipped" + | .skipped.output.title = "${{ inputs.skipped-title }}" + | .skipped.output.summary = "${{ inputs.skipped-summary }}" + + - uses: envoyproxy/toolshed/gh-actions/appauth@actions-v0.2.8 + name: Appauth + id: appauth + with: + app_id: ${{ secrets.app-id }} + key: ${{ secrets.app-key }} + - uses: envoyproxy/toolshed/gh-actions/github/checks@actions-v0.2.8 + name: Start checks + id: checks + with: + checks: ${{ toJSON(fromJSON(inputs.env).checks) }} + config: ${{ steps.check-config.outputs.value }} + text-extra: | + ## ${{ fromJSON(inputs.env).summary.linked-title }} + + ${{ fromJSON(inputs.env).summary.summary }} + token: ${{ steps.appauth.outputs.token }} + - uses: envoyproxy/toolshed/gh-actions/json/table@actions-v0.2.8 + name: Summary + with: + collapse-open: true + json: | + {"checks": ${{ steps.checks.outputs.checks }}, + "config": ${{ toJSON(fromJSON(inputs.env).checks) }}} + filter: | + .checks + heading: >- + ${{ fromJSON(inputs.env).config.envoy.icon }} Checks + mutate-cells: | + .cell as $cell + | .row as $row + | .table as $table + | $cell + | if ($row | index($cell) == 0) then + $table.data.config[$cell].name + elif ($table.data.config[$row[0]].action != "SKIP") then + "[started](http://github.com/${{ github.repository }}/runs/\($cell))" + else "skipped" end + output-path: GITHUB_STEP_SUMMARY + title: Checks started/skipped + + - uses: envoyproxy/toolshed/gh-actions/github/env/save@actions-v0.2.8 + name: Save env + id: data + with: + env: ${{ inputs.env }} + env-filter: | + ${{ steps.checks.outputs.checks }} as $checksStarted + | .checks + |= with_entries( + if $checksStarted[.key] != "skipped" then + .value["check-id"] = $checksStarted[.key] + else . end) diff --git a/.github/workflows/_workflow-start.yml b/.github/workflows/_workflow-start.yml deleted file mode 100644 index 0dcd80bd5aea..000000000000 --- a/.github/workflows/_workflow-start.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Workflow start -# This workflow is only required for externally triggered jobs that need to manually -# set the check status for a commit/PR - -permissions: - contents: read - -on: - workflow_call: - inputs: - workflow_name: - required: true - type: string - sha: - required: true - type: string - -jobs: - start: - runs-on: ubuntu-22.04 - permissions: - contents: read - statuses: write - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/env - id: env - with: - check-mobile-run: false - - - if: ${{ steps.env.outputs.trusted != 'true' }} - name: Start status check - uses: envoyproxy/toolshed/gh-actions/status@actions-v0.1.8 - with: - authToken: ${{ secrets.GITHUB_TOKEN }} - context: ${{ inputs.workflow_name }} - state: 'pending' - sha: ${{ inputs.sha }} - target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - if: ${{ steps.env.outputs.trusted != 'true' }} - name: Save the SHA - env: - STATE_SHA: ${{ inputs.sha }} - run: | - mkdir -p ./sha - echo $STATE_SHA > ./sha/state_sha - - if: ${{ steps.env.outputs.trusted != 'true' }} - uses: actions/upload-artifact@v3 - with: - name: state_sha - path: sha/ diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index 5ec62e9c8afd..03c98f690b0b 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Pre-cleanup - uses: envoyproxy/toolshed/gh-actions/diskspace@actions-v0.1.8 + uses: envoyproxy/toolshed/gh-actions/diskspace@actions-v0.2.8 with: to_remove: | /usr/local/lib/android diff --git a/.github/workflows/command.yml b/.github/workflows/command.yml new file mode 100644 index 000000000000..77cc48f75cc2 --- /dev/null +++ b/.github/workflows/command.yml @@ -0,0 +1,45 @@ +name: Command + +# NB: **ALL** commands should be permissionless and only use an app token or relevant secrets +# specific to their requirements! +permissions: + contents: read + +on: + issue_comment: + types: + - created + + +jobs: + # For speed and _security_ only a single command (first matching) will be parsed/run from a comment + command: + name: Parse and run command + runs-on: ubuntu-22.04 + steps: + - uses: envoyproxy/toolshed/gh-actions/github/command@actions-v0.2.8 + name: Parse command from comment + id: command + with: + text: ${{ github.event.comment.body }} + matching: >- + ^/(retest) + + # /retest + - uses: envoyproxy/toolshed/gh-actions/appauth@actions-v0.2.8 + if: ${{ steps.command.outputs.command == 'retest' }} + id: appauth-retest + name: Appauth (retest) + with: + key: ${{ secrets.ENVOY_CI_APP_KEY }} + app_id: ${{ secrets.ENVOY_CI_APP_ID }} + - uses: envoyproxy/toolshed/gh-actions/retest@actions-v0.2.8 + if: ${{ steps.command.outputs.command == 'retest' }} + name: Retest + with: + token: ${{ steps.appauth-retest.outputs.token }} + azp_org: cncf + azp_token: ${{ secrets.AZP_TOKEN }} + comment-id: ${{ github.event.comment.id }} + pr-url: ${{ github.event.issue.pull_request.url }} + args: ${{ steps.command.outputs.args }} diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml deleted file mode 100644 index d5d25bebf4e1..000000000000 --- a/.github/workflows/commands.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: commands - -on: - issue_comment: - types: - - created - -permissions: - contents: read - -jobs: - retest: - if: >- - ${{ - github.event.issue.pull_request - && github.repository == 'envoyproxy/envoy' - && github.actor != 'repokitteh-read-only[bot]' - && github.actor != 'dependabot[bot]' - }} - name: Retest - runs-on: ubuntu-22.04 - permissions: - pull-requests: write - actions: write - checks: read - steps: - - uses: envoyproxy/toolshed/gh-actions/retest@actions-v0.1.8 - with: - token: ${{ secrets.GITHUB_TOKEN }} - azp_org: cncf - azp_token: ${{ secrets.AZP_TOKEN }} diff --git a/.github/workflows/envoy-dependency.yml b/.github/workflows/envoy-dependency.yml index 5e539e325f34..5fd89dc34296 100644 --- a/.github/workflows/envoy-dependency.yml +++ b/.github/workflows/envoy-dependency.yml @@ -48,15 +48,20 @@ jobs: (${{ inputs.pr && 'PR/' || '' }}${{ inputs.task == 'bazel' && 'bazel' || 'bazel/api' }}/${{ inputs.dependency }}/${{ inputs.version }}) runs-on: ubuntu-22.04 steps: + - id: appauth + name: Appauth + uses: envoyproxy/toolshed/gh-actions/appauth@actions-v0.2.8 + with: + app_id: ${{ secrets.ENVOY_CI_DEP_APP_ID }} + key: ${{ secrets.ENVOY_CI_DEP_APP_KEY }} - id: checkout name: Checkout Envoy repository - uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.1.8 + uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.2.8 with: - app_id: ${{ secrets.ENVOY_CI_DEP_APP_ID }} - app_key: ${{ secrets.ENVOY_CI_DEP_APP_KEY }} + token: ${{ steps.appauth.outputs.token }} - id: version name: Shorten (possible) SHA - uses: envoyproxy/toolshed/gh-actions/str/sub@actions-v0.1.8 + uses: envoyproxy/toolshed/gh-actions/str/sub@actions-v0.2.8 with: string: ${{ inputs.version }} length: 7 @@ -71,13 +76,13 @@ jobs: TARGET: ${{ inputs.task == 'bazel' && 'update' || 'api-update' }} TASK: ${{ inputs.task == 'bazel' && 'bazel' || 'api/bazel' }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: envoyproxy/toolshed/gh-actions/upload/diff@actions-v0.1.8 + - uses: envoyproxy/toolshed/gh-actions/upload/diff@actions-v0.2.8 name: Upload diff with: name: ${{ inputs.dependency }}-${{ steps.version.outputs.string }} - name: Create a PR if: ${{ inputs.pr }} - uses: envoyproxy/toolshed/gh-actions/github/pr@actions-v0.1.8 + uses: envoyproxy/toolshed/gh-actions/github/pr@actions-v0.2.8 with: base: main body: | @@ -106,15 +111,20 @@ jobs: name: Update build image (PR) runs-on: ubuntu-22.04 steps: - - uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.1.8 + - id: appauth + name: Appauth + uses: envoyproxy/toolshed/gh-actions/appauth@actions-v0.2.8 + with: + app_id: ${{ secrets.ENVOY_CI_DEP_APP_ID }} + key: ${{ secrets.ENVOY_CI_DEP_APP_KEY }} + - uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.2.8 id: checkout name: Checkout Envoy repository with: config: | path: envoy fetch-depth: 0 - app_id: ${{ secrets.ENVOY_CI_DEP_APP_ID }} - app_key: ${{ secrets.ENVOY_CI_DEP_APP_KEY }} + token: ${{ steps.appauth.outputs.token }} - uses: actions/checkout@v4 name: Checkout Envoy build tools repository with: @@ -125,39 +135,43 @@ jobs: shas=( tag sha - mobile_sha - gcr_sha) + mobile-sha + gcr-sha) for sha in "${shas[@]}"; do - current_sha=$(bazel run //tools/dependency:build-image-sha "$sha") + current_sha=$(bazel run --config=ci //tools/dependency:build-image-sha "$sha") echo "${sha}=${current_sha}" >> "$GITHUB_OUTPUT" done id: current name: Current SHAs working-directory: envoy - run: | - # get current build image version - CONTAINER_TAG=$(git log -1 --pretty=format:"%H" "./docker") + if [[ -z "$CONTAINER_TAG" ]]; then + # get current build image version + CONTAINER_TAG=$(git log -1 --pretty=format:"%H" "./docker") + fi echo "tag=${CONTAINER_TAG}" >> "$GITHUB_OUTPUT" echo "tag_short=${CONTAINER_TAG::7}" >> "$GITHUB_OUTPUT" + env: + CONTAINER_TAG: ${{ inputs.version }} id: build-tools name: Build image SHA working-directory: build-tools - name: Check Docker SHAs id: build-images - uses: envoyproxy/toolshed/gh-actions/docker/shas@actions-v0.1.8 + uses: envoyproxy/toolshed/gh-actions/docker/shas@actions-v0.2.8 with: images: | sha: envoyproxy/envoy-build-ubuntu:${{ steps.build-tools.outputs.tag }} - mobile_sha: envoyproxy/envoy-build-ubuntu:mobile-${{ steps.build-tools.outputs.tag }} - gcr_sha: gcr.io/envoy-ci/envoy-build:${{ steps.build-tools.outputs.tag }} + mobile-sha: envoyproxy/envoy-build-ubuntu:mobile-${{ steps.build-tools.outputs.tag }} + gcr-sha: gcr.io/envoy-ci/envoy-build:${{ steps.build-tools.outputs.tag }} - run: | SHA_REPLACE=( "$CURRENT_ENVOY_TAG:$ENVOY_TAG" - "$CURRENT_ENVOY_SHA:${OUTPUT_sha}" - "$CURRENT_ENVOY_MOBILE_SHA:${OUTPUT_mobile_sha}" - "$CURRENT_ENVOY_GCR_SHA:${OUTPUT_gcr_sha}") + "$CURRENT_ENVOY_SHA:${{ fromJSON(steps.build-images.outputs.shas).sha }}" + "$CURRENT_ENVOY_MOBILE_SHA:${{ fromJSON(steps.build-images.outputs.shas).mobile-sha }}" + "$CURRENT_ENVOY_GCR_SHA:${{ fromJSON(steps.build-images.outputs.shas).gcr-sha }}") echo "replace=${SHA_REPLACE[*]}" >> "$GITHUB_OUTPUT" name: Find SHAs to replace id: shas @@ -165,16 +179,16 @@ jobs: ENVOY_TAG: ${{ steps.build-tools.outputs.tag }} CURRENT_ENVOY_TAG: ${{ steps.current.outputs.tag }} CURRENT_ENVOY_SHA: ${{ steps.current.outputs.sha }} - CURRENT_ENVOY_MOBILE_SHA: ${{ steps.current.outputs.mobile_sha }} - CURRENT_ENVOY_GCR_SHA: ${{ steps.current.outputs.gcr_sha }} + CURRENT_ENVOY_MOBILE_SHA: ${{ steps.current.outputs.mobile-sha }} + CURRENT_ENVOY_GCR_SHA: ${{ steps.current.outputs.gcr-sha }} - run: | - echo "${SHA_REPLACE}" | xargs bazel run @envoy_toolshed//sha:replace "${PWD}" + echo "${SHA_REPLACE}" | xargs bazel run --config=ci @envoy_toolshed//sha:replace "${PWD}" env: SHA_REPLACE: ${{ steps.shas.outputs.replace }} name: Update SHAs working-directory: envoy - name: Create a PR - uses: envoyproxy/toolshed/gh-actions/github/pr@actions-v0.1.8 + uses: envoyproxy/toolshed/gh-actions/github/pr@actions-v0.2.8 with: base: main body: Created by Envoy dependency bot @@ -186,7 +200,7 @@ jobs: Signed-off-by: ${{ env.COMMITTER_NAME }} <${{ env.COMMITTER_EMAIL }}> title: 'deps: Bump build images -> `${{ steps.build-tools.outputs.tag_short }}`' - GITHUB_TOKEN: ${{ steps.checkout.outputs.token }} + GITHUB_TOKEN: ${{ steps.appauth.outputs.token }} working-directory: envoy scheduled: @@ -208,7 +222,7 @@ jobs: run: | TODAY_DATE=$(date -u -I"date") export TODAY_DATE - bazel run //tools/dependency:check --action_env=TODAY_DATE -- -c release_issues --fix - bazel run //tools/dependency:check --action_env=TODAY_DATE -- -c cves -w error + bazel run --config=ci //tools/dependency:check --action_env=TODAY_DATE -- -c release_issues --fix + bazel run --config=ci //tools/dependency:check --action_env=TODAY_DATE -- -c cves -w error env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/envoy-macos.yml b/.github/workflows/envoy-macos.yml index 6ccae39e8726..f432d866ee82 100644 --- a/.github/workflows/envoy-macos.yml +++ b/.github/workflows/envoy-macos.yml @@ -1,44 +1,58 @@ -name: Envoy/macos +name: Envoy/macOS permissions: contents: read on: - push: - branches: - - main - - release/v* - pull_request_target: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml with: - prime-build-image: false - check-mobile-run: false + cache-docker: false + check-name: macos macos: - needs: - - env - uses: ./.github/workflows/_ci.yml - name: CI ${{ matrix.name || matrix.target }} + permissions: + contents: read + packages: read secrets: rbe-key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + if: ${{ fromJSON(needs.load.outputs.request).run.build-macos }} + needs: + - load + uses: ./.github/workflows/_run.yml + name: CI ${{ matrix.name || matrix.target }} with: - target: ${{ matrix.target }} + command: + container-command: + request: ${{ needs.load.outputs.request }} runs-on: macos-12-xl - command-ci: - command-prefix: - repo-ref: ${{ needs.env.outputs.repo_ref }} steps-post: steps-pre: ${{ matrix.steps-pre }} + target: ${{ matrix.target }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} strategy: fail-fast: false matrix: @@ -59,3 +73,22 @@ jobs: --config=ci --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH}) export BAZEL_BUILD_EXTRA_OPTIONS=${_BAZEL_BUILD_EXTRA_OPTIONS[*]} + + request: + permissions: + actions: read + contents: read + pull-requests: read + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.build-macos }} + needs: + - load + - macos + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/envoy-prechecks.yml b/.github/workflows/envoy-prechecks.yml index 8c6e9157da01..bd8400a1a4ae 100644 --- a/.github/workflows/envoy-prechecks.yml +++ b/.github/workflows/envoy-prechecks.yml @@ -1,47 +1,71 @@ -name: Envoy/prechecks +name: Envoy/Prechecks permissions: contents: read on: - push: - branches: - - main - - release/v* - pull_request_target: - paths: - - '**/requirements*.txt' - - '**/go.mod' - - '**/*.bzl' - - tools/dependency/BUILD - - WORKSPACE - - .github/workflows/envoy-prechecks.yml - - .github/workflows/_*.yml + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true +env: + CI_DEBUG: ${{ vars.CI_DEBUG }} + + jobs: - env: - uses: ./.github/workflows/_env.yml - with: - prime-build-image: true - check-mobile-run: false + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: prechecks deps: - needs: - - env + permissions: + actions: read + contents: read + packages: read + pull-requests: read + name: Precheck (${{ fromJSON(needs.load.outputs.request).summary.title }}) uses: ./.github/workflows/_precheck_deps.yml - name: Precheck ${{ needs.env.outputs.repo-ref-title }} + if: ${{ fromJSON(needs.load.outputs.request).run.precheck-deps }} + needs: + - load with: - build-image-ubuntu: ${{ needs.env.outputs.build_image_ubuntu }} dependency-review: ${{ github.event_name == 'pull_request_target' && github.repository == 'envoyproxy/envoy' }} - repo-ref: ${{ needs.env.outputs.repo-ref }} + request: ${{ needs.load.outputs.request }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read - packages: read + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.precheck-deps }} + needs: + - load + - deps + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/envoy-publish.yml b/.github/workflows/envoy-publish.yml index d5c4178a40eb..05c5048943cf 100644 --- a/.github/workflows/envoy-publish.yml +++ b/.github/workflows/envoy-publish.yml @@ -1,4 +1,7 @@ -name: Publish & verify +# This workflow is triggered by azp currently +# Once arm/x64 build jobs are shifted to github, this can be triggered +# by on: workflow_run +name: Envoy/Publish & verify permissions: contents: read @@ -15,15 +18,25 @@ on: description: "Ref for grouping PRs" concurrency: - group: | + group: >- ${{ github.actor != 'trigger-release-envoy[bot]' && github.event.inputs.head_ref || github.run_id - }}-${{ github.workflow }} + }}-${{ github.event.workflow.id }} cancel-in-progress: true jobs: - env: + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} + permissions: + actions: read + contents: read + packages: read + pull-requests: read if: >- ${{ (github.repository == 'envoyproxy/envoy' @@ -32,57 +45,53 @@ jobs: || github.actor == 'trigger-workflow-envoy[bot]' || github.actor == 'trigger-release-envoy[bot]') }} - uses: ./.github/workflows/_env.yml + uses: ./.github/workflows/_load.yml with: - check-mobile-run: false - prime-build-image: true - repo-ref: ${{ inputs.ref }} - repo-ref-sha: ${{ inputs.sha }} - repo-ref-name: ${{ inputs.head_ref }} - permissions: - contents: read - packages: read - pull-requests: read + check-name: publish + head-sha: ${{ inputs.sha }} - check: - if: ${{ github.event_name != 'pull_request' }} - uses: ./.github/workflows/_workflow-start.yml + publish: permissions: contents: read - statuses: write - with: - workflow_name: Verify/examples - sha: ${{ inputs.sha }} - - publish: + packages: read + if: ${{ fromJSON(needs.load.outputs.request).run.publish }} needs: - - env - - check + - load uses: ./.github/workflows/_stage_publish.yml - name: Publish ${{ needs.env.outputs.repo-ref-title }} + name: Publish with: - build_image_ubuntu: ${{ needs.env.outputs.build_image_ubuntu }} - trusted: ${{ needs.env.outputs.trusted == 'true' && true || false }} - version-dev: ${{ needs.env.outputs.version-dev }} - repo-ref: ${{ inputs.ref }} - sha: ${{ inputs.sha }} + request: ${{ needs.load.outputs.request }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + + verify: permissions: contents: read packages: read - secrets: - ENVOY_CI_SYNC_APP_ID: ${{ secrets.ENVOY_CI_SYNC_APP_ID }} - ENVOY_CI_SYNC_APP_KEY: ${{ secrets.ENVOY_CI_SYNC_APP_KEY }} - ENVOY_CI_PUBLISH_APP_ID: ${{ secrets.ENVOY_CI_PUBLISH_APP_ID }} - ENVOY_CI_PUBLISH_APP_KEY: ${{ secrets.ENVOY_CI_PUBLISH_APP_KEY }} - - verify: - uses: ./.github/workflows/_stage_verify.yml - name: Verify ${{ needs.env.outputs.repo-ref-title }} + if: ${{ fromJSON(needs.load.outputs.request).run.verify }} needs: - - env + - load + uses: ./.github/workflows/_stage_verify.yml + name: Verify + with: + request: ${{ needs.load.outputs.request }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read + pull-requests: read + if: >- + ${{ always() + && (fromJSON(needs.load.outputs.request).run.publish + || fromJSON(needs.load.outputs.request).run.verify) }} + needs: + - load + - publish + - verify + uses: ./.github/workflows/_finish.yml with: - trusted: ${{ needs.env.outputs.trusted == 'true' && true || false }} - given_ref: ${{ inputs.ref }} - repo-ref: ${{ inputs.ref }} + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/envoy-release.yml b/.github/workflows/envoy-release.yml index 78511d9df705..23d8b13da867 100644 --- a/.github/workflows/envoy-release.yml +++ b/.github/workflows/envoy-release.yml @@ -55,7 +55,7 @@ jobs: steps: - id: checkout name: Checkout Envoy repository - uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.1.8 + uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.2.8 with: app_id: ${{ secrets.ENVOY_CI_PUBLISH_APP_ID }} app_key: ${{ secrets.ENVOY_CI_PUBLISH_APP_KEY }} @@ -77,10 +77,10 @@ jobs: name: Check changelog summary - if: ${{ inputs.author }} name: Validate signoff email - uses: envoyproxy/toolshed/gh-actions/email/validate@actions-v0.1.8 + uses: envoyproxy/toolshed/gh-actions/email/validate@actions-v0.2.8 with: email: ${{ inputs.author }} - - uses: envoyproxy/toolshed/gh-actions/github/run@actions-v0.1.8 + - uses: envoyproxy/toolshed/gh-actions/github/run@actions-v0.2.8 name: Create release with: source: | @@ -105,7 +105,7 @@ jobs: name: Release version id: release - name: Create a PR - uses: envoyproxy/toolshed/gh-actions/github/pr@actions-v0.1.8 + uses: envoyproxy/toolshed/gh-actions/github/pr@actions-v0.2.8 with: base: ${{ github.ref_name }} commit: false @@ -113,8 +113,8 @@ jobs: body: | Created by Envoy publish bot for @${{ github.actor }} ${{ ! inputs.summary && ':warning: Created without changelog summary, this will need to be updated before publishing' || '' }} - branch: release/create/${{ steps.checkout.outputs.branch_name }} - diff-upload: release-${{ steps.checkout.outputs.branch_name }} + branch: release/create/${{ steps.checkout.outputs.branch-name }} + diff-upload: release-${{ steps.checkout.outputs.branch-name }} diff-show: true dry-run: ${{ ! inputs.pr }} wip: ${{ ! inputs.summary || inputs.wip }} @@ -130,13 +130,13 @@ jobs: steps: - id: checkout name: Checkout Envoy repository - uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.1.8 + uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.2.8 with: app_id: ${{ secrets.ENVOY_CI_PUBLISH_APP_ID }} app_key: ${{ secrets.ENVOY_CI_PUBLISH_APP_KEY }} committer-name: ${{ env.COMMITTER_NAME }} committer-email: ${{ env.COMMITTER_EMAIL }} - - uses: envoyproxy/toolshed/gh-actions/github/run@actions-v0.1.8 + - uses: envoyproxy/toolshed/gh-actions/github/run@actions-v0.2.8 name: Sync version histories with: command: >- @@ -146,15 +146,15 @@ jobs: -- --signoff="${{ env.COMMITTER_NAME }} <${{ env.COMMITTER_EMAIL }}>" - name: Create a PR - uses: envoyproxy/toolshed/gh-actions/github/pr@actions-v0.1.8 + uses: envoyproxy/toolshed/gh-actions/github/pr@actions-v0.2.8 with: append-commit-message: true base: ${{ github.ref_name }} commit: false body: | Created by Envoy publish bot for @${{ github.actor }} - branch: release/sync/${{ steps.checkout.outputs.branch_name }} - diff-upload: version-histories-${{ steps.checkout.outputs.branch_name }} + branch: release/sync/${{ steps.checkout.outputs.branch-name }} + diff-upload: version-histories-${{ steps.checkout.outputs.branch-name }} diff-show: true dry-run: ${{ ! inputs.pr }} GITHUB_TOKEN: ${{ steps.checkout.outputs.token }} @@ -176,7 +176,7 @@ jobs: name: Create release branch steps: - name: Checkout repository - uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.1.8 + uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.2.8 with: app_id: ${{ secrets.ENVOY_CI_PUBLISH_APP_ID }} app_key: ${{ secrets.ENVOY_CI_PUBLISH_APP_KEY }} diff --git a/.github/workflows/envoy-sync.yml b/.github/workflows/envoy-sync.yml index bf79e682d540..1daafe4a8b03 100644 --- a/.github/workflows/envoy-sync.yml +++ b/.github/workflows/envoy-sync.yml @@ -28,10 +28,14 @@ jobs: - data-plane-api - mobile-website steps: - - uses: envoyproxy/toolshed/gh-actions/dispatch@actions-v0.1.8 + - uses: envoyproxy/toolshed/gh-actions/appauth@actions-v0.2.8 + id: appauth + with: + app_id: ${{ secrets.ENVOY_CI_SYNC_APP_ID }} + key: ${{ secrets.ENVOY_CI_SYNC_APP_KEY }} + - uses: envoyproxy/toolshed/gh-actions/dispatch@actions-v0.2.8 with: repository: "envoyproxy/${{ matrix.downstream }}" ref: main - key: "${{ secrets.ENVOY_CI_SYNC_APP_KEY }}" + token: ${{ steps.appauth.outputs.token }} workflow: envoy-sync.yaml - app_id: ${{ secrets.ENVOY_CI_SYNC_APP_ID }} diff --git a/.github/workflows/envoy-windows.yml b/.github/workflows/envoy-windows.yml index ca45b69e7d80..21d9a992da25 100644 --- a/.github/workflows/envoy-windows.yml +++ b/.github/workflows/envoy-windows.yml @@ -1,51 +1,53 @@ -name: Envoy/windows +name: Envoy/Windows permissions: contents: read on: - push: - branches: - - main - - release/v* - pull_request_target: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml with: - prime-build-image: false - check-mobile-run: false + cache-docker: false + check-name: windows windows: - needs: - - env - strategy: - fail-fast: false - matrix: - include: - - target: ci/windows_ci_steps.sh - name: Windows 2019 - uses: ./.github/workflows/_ci.yml - name: CI ${{ matrix.name || matrix.target }} + permissions: + contents: read + packages: read secrets: rbe-key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + if: ${{ fromJSON(needs.load.outputs.request).run.build-windows }} + needs: + - load + uses: ./.github/workflows/_run.yml + name: CI ${{ matrix.name || matrix.target }} with: - target: ${{ matrix.target }} + command: + request: ${{ needs.load.outputs.request }} runs-on: envoy-win19-small - command-ci: - repo-ref: ${{ needs.env.outputs.repo-ref }} - steps-post: - temp-dir: 'C:\Users\runner\AppData\Local\Temp\bazel-shared' - upload-name: windows.release - upload-path: 'C:\Users\runner\AppData\Local\Temp\envoy' source: | export ENVOY_SHARED_TMP_DIR="C:\Users\runner\AppData\Local\Temp\bazel-shared" export ENVOY_DOCKER_BUILD_DIR="C:\Users\runner\AppData\Local\Temp" @@ -60,10 +62,22 @@ jobs: --jobs=75 --flaky_test_attempts=2) export BAZEL_BUILD_EXTRA_OPTIONS=${_BAZEL_BUILD_EXTRA_OPTIONS[*]} + steps-post: + target: ${{ matrix.target }} + temp-dir: 'C:\Users\runner\AppData\Local\Temp\bazel-shared' + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + upload-name: windows.release + upload-path: 'C:\Users\runner\AppData\Local\Temp\envoy' + strategy: + fail-fast: false + matrix: + include: + - target: ci/windows_ci_steps.sh + name: Windows 2019 docker: needs: - - env + - load - windows strategy: fail-fast: false @@ -85,7 +99,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ needs.env.outputs.repo_ref }} + ref: ${{ needs.load.outputs.repo_ref }} - uses: actions/download-artifact@v3 with: name: windows.release @@ -99,8 +113,27 @@ jobs: shell: bash env: CI_BRANCH: ${{ github.ref }} - DOCKERHUB_USERNAME: ${{ needs.env.outputs.trusted == 'true' && secrets.DOCKERHUB_USERNAME || '' }} - DOCKERHUB_PASSWORD: ${{ needs.env.outputs.trusted == 'true' && secrets.DOCKERHUB_PASSWORD || '' }} + DOCKERHUB_USERNAME: ${{ fromJSON(needs.load.outputs.trusted) && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_PASSWORD: ${{ fromJSON(needs.load.outputs.trusted) && secrets.DOCKERHUB_PASSWORD || '' }} WINDOWS_BUILD_TYPE: ${{ matrix.build-type }} WINDOWS_IMAGE_BASE: ${{ matrix.image-base }} WINDOWS_IMAGE_TAG: ${{ matrix.image-tag }} + request: + permissions: + actions: read + contents: read + pull-requests: read + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.build-windows }} + needs: + - load + - windows + - docker + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-android_build.yml b/.github/workflows/mobile-android_build.yml index 7b6d23acac20..b886cafbce4e 100644 --- a/.github/workflows/mobile-android_build.yml +++ b/.github/workflows/mobile-android_build.yml @@ -1,291 +1,188 @@ -name: android_build +name: Mobile/Android build permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-android - androidbuild: - if: ${{ needs.env.outputs.mobile_android_build == 'true' }} - needs: env + build: permissions: contents: read packages: read - name: android_build - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 90 - container: - image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} - env: - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Build envoy.aar distributable' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-release-clang \ - --fat_apk_cpu=x86_64 \ - --linkopt=-fuse-ld=lld \ - //:android_dist + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-android }} + needs: load + name: Build envoy.aar distributable + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: >- + build + --config=mobile-remote-release-clang + --fat_apk_cpu=x86_64 + --linkopt=-fuse-ld=lld + //:android_dist + container: ${{ fromJSON(needs.load.outputs.build-image).mobile }} + diskspace-hack: true + request: ${{ needs.load.outputs.request }} + timeout-minutes: 90 + target: build - javahelloworld: - if: ${{ needs.env.outputs.mobile_android_build_all == 'true' }} - needs: - - env - - androidbuild + kotlin-hello-world: permissions: contents: read packages: read - name: java_helloworld - runs-on: envoy-x64-small - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 - with: - java-version: '11' - java-package: jdk - architecture: x64 - distribution: zulu - - name: 'Install dependencies' - run: | - cd mobile - ./ci/linux_ci_setup.sh - # https://github.blog/changelog/2023-02-23-hardware-accelerated-android-virtualization-on-actions-windows-and-linux-larger-hosted-runners/ - - name: Enable KVM group permissions - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start emulator' - with: - timeout_minutes: 15 - max_attempts: 1 - command: ./mobile/ci/start_android_emulator.sh - # Return to using: - # cd mobile && ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/java/hello_world:hello_envoy - # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - - name: 'Start Java app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-release-clang \ - --fat_apk_cpu=x86_64 \ - --linkopt=-fuse-ld=lld \ - //examples/java/hello_world:hello_envoy - "${ANDROID_HOME}"/platform-tools/adb install -r --no-incremental bazel-bin/examples/java/hello_world/hello_envoy.apk - "${ANDROID_HOME}"/platform-tools/adb shell am start -n io.envoyproxy.envoymobile.helloenvoy/.MainActivity - - name: 'Check connectivity' - run: | - timeout 30 "${ANDROID_HOME}"/platform-tools/adb logcat -e "received headers with status 301" -m 1 || { - echo "Failed checking for headers in adb logcat" >&2 - timeout 30 "${ANDROID_HOME}"/platform-tools/adb logcat || { - echo "Failed dumping adb logcat" >&2 - } - exit 1 - } - - kotlinhelloworld: - if: ${{ needs.env.outputs.mobile_android_build == 'true' }} + name: kotlin-hello-world + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-android }} needs: - - env - - androidbuild - permissions: - contents: read - packages: read - name: kotlin_helloworld - runs-on: envoy-x64-small - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 - with: - java-version: '11' - java-package: jdk - architecture: x64 - distribution: zulu - - name: 'Install dependencies' - run: | - cd mobile - ./ci/linux_ci_setup.sh - # https://github.blog/changelog/2023-02-23-hardware-accelerated-android-virtualization-on-actions-windows-and-linux-larger-hosted-runners/ - - name: Enable KVM group permissions - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start emulator' - with: - timeout_minutes: 15 - max_attempts: 1 - command: ./mobile/ci/start_android_emulator.sh - # Return to using: - # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt - # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - - name: 'Start Kotlin app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-release-clang \ - --fat_apk_cpu=x86_64 \ - --linkopt=-fuse-ld=lld \ - //examples/kotlin/hello_world:hello_envoy_kt - "${ANDROID_HOME}"/platform-tools/adb install -r --no-incremental bazel-bin/examples/kotlin/hello_world/hello_envoy_kt.apk - "${ANDROID_HOME}"/platform-tools/adb shell am start -n io.envoyproxy.envoymobile.helloenvoykotlin/.MainActivity - - name: 'Check connectivity' - run: | - timeout 30 "${ANDROID_HOME}"/platform-tools/adb logcat -e "received headers with status 200" -m 1 || { - echo "Failed checking for headers in adb logcat" >&2 - timeout 30 "${ANDROID_HOME}"/platform-tools/adb logcat || { - echo "Failed dumping adb logcat" >&2 - } - exit 1 - } + - load + - build + with: + command: ./bazelw + container-command: + # Return to using: + # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt + # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. + args: >- + build + --config=mobile-remote-release-clang + --fat_apk_cpu=x86_64 + --linkopt=-fuse-ld=lld + //examples/kotlin/hello_world:hello_envoy_kt + request: ${{ needs.load.outputs.request }} + target: kotlin-hello-world + runs-on: envoy-x64-small + steps-pre: | + - uses: envoyproxy/toolshed/gh-actions/envoy/android/pre@actions-v0.2.8 + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/android/post@actions-v0.2.8 + with: + apk: bazel-bin/examples/kotlin/hello_world/hello_envoy_kt.apk + app: io.envoyproxy.envoymobile.helloenvoykotlin/.MainActivity + status: 200 + timeout-minutes: 50 + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile - kotlinbaselineapp: - if: ${{ needs.env.outputs.mobile_android_build_all == 'true' }} - needs: - - env - - androidbuild + apps: permissions: contents: read packages: read - name: kotlin_baseline_app - runs-on: envoy-x64-small - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 - with: - java-version: '11' - java-package: jdk - architecture: x64 - distribution: zulu - - name: 'Install dependencies' - run: | - cd mobile - ./ci/linux_ci_setup.sh - # https://github.blog/changelog/2023-02-23-hardware-accelerated-android-virtualization-on-actions-windows-and-linux-larger-hosted-runners/ - - name: Enable KVM group permissions - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start emulator' - with: - timeout_minutes: 15 - max_attempts: 1 - command: ./mobile/ci/start_android_emulator.sh - # Return to using: - # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt - # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - - name: 'Start Kotlin app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-release-clang \ - --fat_apk_cpu=x86_64 \ - --linkopt=-fuse-ld=lld \ + name: Android apps + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-android-all }} + needs: + - load + - build + with: + command: ./bazelw + container-command: + args: ${{ matrix.args }} + request: ${{ needs.load.outputs.request }} + target: ${{ matrix.target }} + runs-on: envoy-x64-small + steps-pre: | + - uses: envoyproxy/toolshed/gh-actions/envoy/android/pre@actions-v0.2.8 + steps-post: ${{ matrix.steps-post }} + timeout-minutes: 50 + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: java-hello-world + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/android/post@actions-v0.2.8 + with: + apk: bazel-bin/examples/java/hello_world/hello_envoy.apk + app: io.envoyproxy.envoymobile.helloenvoy/.MainActivity + status: 301 + target: java-hello-world + args: >- + build + --config=mobile-remote-release-clang + --fat_apk_cpu=x86_64 + --linkopt=-fuse-ld=lld + //examples/java/hello_world:hello_envoy + - name: kotlin-baseline-app + # Return to using: + # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt + # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. + args: >- + build + --config=mobile-remote-release-clang + --fat_apk_cpu=x86_64 + --linkopt=-fuse-ld=lld //test/kotlin/apps/baseline:hello_envoy_kt - "${ANDROID_HOME}"/platform-tools/adb install -r --no-incremental bazel-bin/test/kotlin/apps/baseline/hello_envoy_kt.apk - "${ANDROID_HOME}"/platform-tools/adb shell am start -n io.envoyproxy.envoymobile.helloenvoybaselinetest/.MainActivity - - name: 'Check connectivity' - run: | - timeout 30 "${ANDROID_HOME}"/platform-tools/adb logcat -e "received headers with status 301" -m 1 || { - echo "Failed checking for headers in adb logcat" >&2 - timeout 30 "${ANDROID_HOME}"/platform-tools/adb logcat || { - echo "Failed dumping adb logcat" >&2 - } - exit 1 - } + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/android/post@actions-v0.2.8 + with: + apk: bazel-bin/test/kotlin/apps/baseline/hello_envoy_kt.apk + app: io.envoyproxy.envoymobile.helloenvoybaselinetest/.MainActivity + status: 301 + target: kotlin-baseline-app + - name: kotlin-experimental-app + # Return to using: + # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt + # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. + args: >- + build + --config=mobile-remote-release-clang + --fat_apk_cpu=x86_64 + --linkopt=-fuse-ld=lld + //test/kotlin/apps/experimental:hello_envoy_kt + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/android/post@actions-v0.2.8 + with: + apk: bazel-bin/test/kotlin/apps/experimental/hello_envoy_kt.apk + app: io.envoyproxy.envoymobile.helloenvoyexperimentaltest/.MainActivity + status: 200 + target: kotlin-experimental-app - kotlinexperimentalapp: - if: ${{ needs.env.outputs.mobile_android_build_all == 'true' }} - needs: - - env - - androidbuild + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read - packages: read - name: kotlin_experimental_app - runs-on: envoy-x64-small - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 - with: - java-version: '11' - java-package: jdk - architecture: x64 - distribution: zulu - - name: 'Install dependencies' - run: | - cd mobile - ./ci/linux_ci_setup.sh - # https://github.blog/changelog/2023-02-23-hardware-accelerated-android-virtualization-on-actions-windows-and-linux-larger-hosted-runners/ - - name: Enable KVM group permissions - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start emulator' - with: - timeout_minutes: 15 - max_attempts: 1 - command: ./mobile/ci/start_android_emulator.sh - # Return to using: - # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt - # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - - name: 'Start Kotlin app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-release-clang \ - --fat_apk_cpu=x86_64 \ - --linkopt=-fuse-ld=lld \ - //test/kotlin/apps/experimental:hello_envoy_kt - "${ANDROID_HOME}"/platform-tools/adb install -r --no-incremental bazel-bin/test/kotlin/apps/experimental/hello_envoy_kt.apk - "${ANDROID_HOME}"/platform-tools/adb shell am start -n io.envoyproxy.envoymobile.helloenvoyexperimentaltest/.MainActivity - - name: 'Check connectivity' - run: | - timeout 30 "${ANDROID_HOME}"/platform-tools/adb logcat -e "received headers with status 200" -m 1 || { - echo "Failed checking for headers in adb logcat" >&2 - timeout 30 "${ANDROID_HOME}"/platform-tools/adb logcat || { - echo "Failed dumping adb logcat" >&2 - } - exit 1 - } + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-android }} + needs: + - load + - build + - kotlin-hello-world + - apps + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-android_tests.yml b/.github/workflows/mobile-android_tests.yml index 16704c01cef7..c513a648d2c8 100644 --- a/.github/workflows/mobile-android_tests.yml +++ b/.github/workflows/mobile-android_tests.yml @@ -1,89 +1,90 @@ -name: android_tests +name: Mobile/Android tests permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - uses: ./.github/workflows/_env.yml - with: - prime-build-image: true + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-android-tests - javatestslinux: - if: ${{ needs.env.outputs.mobile_android_tests == 'true' }} - needs: env + linux: permissions: contents: read packages: read - name: java_tests_linux - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 90 - steps: - - name: Pre-cleanup - # Using the defaults in - # https://github.com/envoyproxy/toolshed/blob/main/gh-actions/diskspace/action.yml. - uses: envoyproxy/toolshed/gh-actions/diskspace@actions-v0.1.8 - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Run Kotlin library integration tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - run: | - docker run --volume="${PWD}:/source" --workdir="/source/mobile" \ - -e GITHUB_TOKEN -e CC -e CXX ${{ needs.env.outputs.build_image_ubuntu_mobile }} \ - ./bazelw test \ - --build_tests_only \ - --config=test-android \ - --config=mobile-remote-ci \ - --define=signal_trace=disabled \ - //test/java/... + name: Android linux tests + uses: ./.github/workflows/_mobile_container_ci.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-android-tests }} + needs: load + with: + args: ${{ matrix.args }} + container: ${{ fromJSON(needs.load.outputs.build-image).mobile }} + diskspace-hack: true + request: ${{ needs.load.outputs.request }} + target: ${{ matrix.target }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - name: java + target: java_tests_linux + args: >- + test + --build_tests_only + --config=test-android + --config=mobile-remote-ci + --define=signal_trace=disabled + //test/java/... + - name: kotlin + target: kotlin_tests_linux + args: >- + test + --build_tests_only + --config=test-android + --config=mobile-remote-ci + --define=signal_trace=disabled + //test/kotlin/... - kotlintestslinux: - if: ${{ needs.env.outputs.mobile_android_tests == 'true' }} - needs: env + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read - packages: read - # Only kotlin tests are executed since with linux: - # https://github.com/envoyproxy/envoy-mobile/issues/1418. - name: kotlin_tests_linux - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 90 - steps: - - name: Pre-cleanup - # Using the defaults in - # https://github.com/envoyproxy/toolshed/blob/main/gh-actions/diskspace/action.yml. - uses: envoyproxy/toolshed/gh-actions/diskspace@actions-v0.1.8 - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Run Kotlin library integration tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - run: | - docker run --volume="${PWD}:/source" --workdir="/source/mobile" \ - -e GITHUB_TOKEN -e CC -e CXX ${{ needs.env.outputs.build_image_ubuntu_mobile }} \ - ./bazelw test \ - --build_tests_only \ - --config=test-android \ - --config=mobile-remote-ci \ - --define=signal_trace=disabled \ - //test/kotlin/... + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-android-tests }} + needs: + - load + - linux + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-asan.yml b/.github/workflows/mobile-asan.yml index 5faace5c24e1..f83cc183c188 100644 --- a/.github/workflows/mobile-asan.yml +++ b/.github/workflows/mobile-asan.yml @@ -1,50 +1,69 @@ -name: mobile_asan +name: Mobile/ASAN permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-asan asan: - if: ${{ needs.env.outputs.mobile_asan == 'true' }} - needs: env permissions: contents: read packages: read name: asan - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 180 - container: - image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} - env: - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Run tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw test \ - --test_env=ENVOY_IP_TEST_VERSIONS=v4only \ - --config=mobile-remote-ci-linux-asan \ - //test/common/... + uses: ./.github/workflows/_mobile_container_ci.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-asan }} + needs: load + with: + args: >- + test + --test_env=ENVOY_IP_TEST_VERSIONS=v4only + --config=mobile-remote-ci-linux-asan + //test/common/... + request: ${{ needs.load.outputs.request }} + target: asan + timeout-minutes: 180 + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-asan }} + needs: + - load + - asan + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-cc_tests.yml b/.github/workflows/mobile-cc_tests.yml index 2a37430af34d..14f971372c88 100644 --- a/.github/workflows/mobile-cc_tests.yml +++ b/.github/workflows/mobile-cc_tests.yml @@ -1,49 +1,70 @@ -name: mobile_cc_tests +name: Mobile/CC tests permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-cc - cctests: - if: ${{ needs.env.outputs.mobile_cc_tests == 'true' }} - needs: env + cc-tests: permissions: contents: read packages: read - name: cc_tests - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 120 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: 'Run tests' - # Regression test using the new API listener. TODO(#2711) clean up. - run: | - cd mobile - ./bazelw test \ - --action_env=LD_LIBRARY_PATH \ - --copt=-DUSE_API_LISTENER \ - --config=mobile-remote-ci \ - //test/cc/... + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-cc }} + needs: load + name: cc-tests + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: >- + test + --action_env=LD_LIBRARY_PATH + --copt=-DUSE_API_LISTENER + --config=mobile-remote-ci + //test/cc/... + request: ${{ needs.load.outputs.request }} + target: cc-tests + timeout-minutes: 120 + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-cc }} + needs: + - load + - cc-tests + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml index 6342f0dc62c3..77d07e6442aa 100644 --- a/.github/workflows/mobile-compile_time_options.yml +++ b/.github/workflows/mobile-compile_time_options.yml @@ -1,177 +1,174 @@ -name: mobile_compile_time_options +name: Mobile/Compile time options permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-compile-time-options - cc_test_no_yaml: - needs: env + cc: permissions: contents: read packages: read - name: cc_test_no_yaml - runs-on: ubuntu-20.04 - timeout-minutes: 120 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Running C++ test with YAML disabled' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Envoy Mobile build which verifies that the build configuration where YAML is disabled. - run: | - cd mobile - ./bazelw test \ - --config=mobile-remote-ci \ - --define=envoy_yaml=disabled \ - --define=envoy_full_protos=disabled \ - --test_env=ENVOY_IP_TEST_VERSIONS=v4only \ + uses: ./.github/workflows/_mobile_container_ci.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-compile-time-options }} + needs: load + with: + args: ${{ matrix.args }} + command: ./bazelw + entrypoint: ${{ matrix.entrypoint }} + request: ${{ needs.load.outputs.request }} + target: ${{ matrix.target }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - name: Running C++ test with YAML disabled + target: cc-test-no-yaml + args: >- + build + --config=mobile-remote-ci + --define=envoy_yaml=disabled + --define=envoy_full_protos=disabled + --test_env=ENVOY_IP_TEST_VERSIONS=v4only //test/common/integration:client_integration_test - cc_build_no_exceptions: - needs: env - permissions: - contents: read - packages: read - name: cc_test_no_yaml - runs-on: ubuntu-20.04 - timeout-minutes: 120 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Running C++ build with exceptions disabled' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Envoy Mobile build which verifies that the build configuration where YAML is disabled. - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-ci \ - --define envoy_exceptions=disabled \ - --define=envoy_yaml=disabled \ - --copt=-fno-unwind-tables \ - --copt=-fno-exceptions \ - --define=google_grpc=disabled \ - --define=envoy_mobile_xds=disabled \ - //test/performance:test_binary_size //library/cc/... + - name: Running C++ build with exceptions disabled + target: cc-no-build-exceptions + args: >- + build + --config=mobile-remote-ci + --define envoy_exceptions=disabled + --define=envoy_yaml=disabled + --copt=-fno-unwind-tables + --copt=-fno-exceptions + --define=google_grpc=disabled + --define=envoy_mobile_xds=disabled + //test/performance:test_binary_size //library/cc/... + - name: Running C++ test + target: cc-test + args: >- + test + --test_output=all + --config=mobile-remote-ci + --define=signal_trace=disabled + --define=google_grpc=disabled + --define=envoy_mobile_xds=disabled + --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= + entrypoint: | + #!/bin/bash -e + export PATH=/opt/llvm/bin:$PATH + cd /source/mobile + EXTRA_ARGS=$(bazel query //test/cc/... + //test/common/... except //test/common/integration:client_integration_test) + exec "$@" $EXTRA_ARGS - cc_test: - needs: env + build: permissions: contents: read packages: read - name: cc_test - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 120 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Running C++ tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - TARGETS=$(bazel query --noshow_progress --noshow_loading_progress //test/cc/... + //test/common/... except //test/common/integration:client_integration_test) - ./bazelw test \ - --test_output=all \ - --config=mobile-remote-ci \ - --define=signal_trace=disabled \ - --define=google_grpc=disabled \ - --define=envoy_mobile_xds=disabled \ - --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ - $TARGETS - - swift_build: - if: ${{ needs.env.outputs.mobile_compile_time_options == 'true' }} - needs: env - permissions: - contents: read - packages: read - name: swift_build - runs-on: macos-12 - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile - ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build Swift library' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw shutdown - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ - --define=signal_trace=disabled \ - --define=envoy_mobile_stats_reporting=disabled \ - --define=envoy_mobile_swift_cxx_interop=disabled \ - --define=google_grpc=disabled \ - --define=envoy_mobile_xds=disabled \ - --@envoy//bazel:http3=False \ - --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-compile-time-options }} + needs: load + with: + args: ${{ matrix.args }} + command: ./bazelw + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: macos-12 + source: ${{ matrix.source }} + steps-pre: ${{ matrix.steps-pre }} + target: ${{ matrix.target || matrix.name }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + timeout-minutes: 120 + working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: kotlin-build + args: >- + build + --config=mobile-remote-ci-macos + --fat_apk_cpu=x86_64 + --define=signal_trace=disabled + --define=envoy_mobile_request_compression=disabled + --define=envoy_enable_http_datagrams=disabled + --define=google_grpc=disabled + --define=envoy_mobile_xds=disabled + --define=envoy_yaml=disabled + --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= + //:android_dist + source: | + . ./ci/mac_ci_setup.sh --android + echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}" >> $GITHUB_ENV + export ANDROID_NDK_HOME + steps-pre: | + - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 + with: + java-version: '8' + java-package: jdk + architecture: x64 + distribution: zulu + - name: swift-build + args: >- + build + --config=ios + --config=mobile-remote-ci-macos + --define=signal_trace=disabled + --define=envoy_mobile_request_compression=disabled + --define=envoy_mobile_stats_reporting=disabled + --define=envoy_mobile_swift_cxx_interop=disabled + --define=google_grpc=disabled + --define=envoy_mobile_xds=disabled + --@envoy//bazel:http3=False + --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= //library/swift:ios_framework + source: | + ./ci/mac_ci_setup.sh + ./bazelw shutdown - kotlin_build: - if: ${{ needs.env.outputs.mobile_compile_time_options == 'true' }} - needs: env + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read - packages: read - name: kotlin_build - runs-on: macos-12 - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 - with: - java-version: '8' - java-package: jdk - architecture: x64 - distribution: zulu - - name: 'Install dependencies' - run: | - cd mobile - ./ci/mac_ci_setup.sh --android - - name: 'Build Kotlin library' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-ci-macos \ - --fat_apk_cpu=x86_64 \ - --define=signal_trace=disabled \ - --define=envoy_enable_http_datagrams=disabled \ - --define=google_grpc=disabled \ - --define=envoy_mobile_xds=disabled \ - --define=envoy_yaml=disabled \ - --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ - //:android_dist + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-compile-time-options }} + needs: + - load + - cc + - build + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-core.yml b/.github/workflows/mobile-core.yml index 241bcb056612..2d7e9550d9e2 100644 --- a/.github/workflows/mobile-core.yml +++ b/.github/workflows/mobile-core.yml @@ -1,49 +1,70 @@ -name: mobile_core +name: Mobile/Core permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-core - unittests: - if: ${{ github.repository == 'envoyproxy/envoy' }} - needs: env + unit-tests: permissions: contents: read packages: read - name: unit_tests - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 120 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Run tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw test \ - --build_tests_only \ - --action_env=LD_LIBRARY_PATH \ - --test_env=ENVOY_IP_TEST_VERSIONS=v4only \ - --config=mobile-remote-ci \ - //test/common/... + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-core }} + needs: load + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: >- + test + --build_tests_only + --action_env=LD_LIBRARY_PATH + --test_env=ENVOY_IP_TEST_VERSIONS=v4only + --config=mobile-remote-ci + //test/common/... + request: ${{ needs.load.outputs.request }} + target: unit-tests + timeout-minutes: 120 + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-core }} + needs: + - load + - unit-tests + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-coverage.yml b/.github/workflows/mobile-coverage.yml index bd7b7214a990..160d36e91ed0 100644 --- a/.github/workflows/mobile-coverage.yml +++ b/.github/workflows/mobile-coverage.yml @@ -1,59 +1,79 @@ -name: mobile_coverage +name: Mobile/Coverage permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-coverage coverage: - if: ${{ needs.env.outputs.mobile_coverage == 'true' }} - needs: env permissions: contents: read packages: read - name: coverage - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 120 - defaults: - run: - shell: bash - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Run coverage' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - export BAZEL_BUILD_OPTION_LIST="--config=mobile-remote-ci-linux-coverage" \ - PATH=/opt/llvm/bin:${PATH} \ - COVERAGE_THRESHOLD=76 - ../test/run_envoy_bazel_coverage.sh //test/common/... //test/cc/... - - name: 'Package coverage' - run: | - cd mobile - tar -czf coverage.tar.gz generated/coverage - - name: 'Upload report' - uses: actions/upload-artifact@v3 - with: - name: coverage.tar.gz - path: mobile/coverage.tar.gz + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-coverage }} + needs: load + name: Running mobile coverage + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: >- + //test/common/... + //test/cc/... + command: ../test/run_envoy_bazel_coverage.sh + request: ${{ needs.load.outputs.request }} + source: + export COVERAGE_THRESHOLD=76 + export BAZEL_BUILD_OPTION_LIST=--config=mobile-remote-ci-linux-coverage + steps-post: | + - name: Package coverage + shell: bash + run: | + cd mobile + tar -czf coverage.tar.gz generated/coverage + target: mobile-coverage + timeout-minutes: 120 + upload-name: coverage.tar.gz + upload-path: mobile/coverage.tar.gz + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-coverage }} + needs: + - load + - coverage + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-docs.yml b/.github/workflows/mobile-docs.yml index d4d226946ede..0c3268de6bc1 100644 --- a/.github/workflows/mobile-docs.yml +++ b/.github/workflows/mobile-docs.yml @@ -1,63 +1,96 @@ -name: mobile_docs +name: Mobile/Docs permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-docs docs: - if: ${{ github.repository == 'envoyproxy/envoy' }} - needs: env + secrets: + ssh-key-extra: ${{ needs.load.outputs.trusted && secrets.ENVOY_MOBILE_WEBSITE_DEPLOY_KEY || '' }} permissions: contents: read packages: read - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 20 - steps: - # Checkout the Envoy repo - - uses: actions/checkout@v4 - # Checkout the envoy-mobile/envoy-mobile.github.io repo - - uses: actions/checkout@v4 - with: + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-docs }} + needs: load + uses: ./.github/workflows/_run.yml + with: + args: >- + -- + command: ./docs/build.sh + request: ${{ needs.load.outputs.request }} + target: mobile-docs + cache-build-image: ${{ fromJSON(needs.load.outputs.build-image).build-image }} + checkout-extra: | repository: envoy-mobile/envoy-mobile.github.io path: mobile-docs fetch-depth: 0 - ssh-key: ${{ secrets.ENVOY_MOBILE_WEBSITE_DEPLOY_KEY }} - - name: Add safe directory - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - # Not sure if this is necessary - - name: Add safe directory - run: git config --global --add safe.directory "$GITHUB_WORKSPACE/mobile-docs" - - name: Generate docs - run: ./ci/run_envoy_docker.sh 'cd mobile && docs/build.sh' - - name: Publish docs - run: ./ci/run_envoy_docker.sh 'cd mobile && docs/publish.sh' - env: + branch: master + source: | + echo "ENVOY_DOCKER_SOURCE_DIR=/source/mobile" >> $GITHUB_ENV # Path relative to ./mobile directory - MOBILE_DOCS_CHECKOUT_DIR: ../mobile-docs - # This step needs to be done outside the container to access ssh creds - - name: Push changes - if: ${{ github.event_name != 'pull_request' }} - run: | - git -C mobile-docs push origin master + echo "MOBILE_DOCS_CHECKOUT_DIR=../mobile-docs" >> $GITHUB_ENV + steps-post: | + - name: Publish docs + shell: bash + run: | + ./ci/run_envoy_docker.sh docs/publish.sh + # This step needs to be done outside the container to access ssh creds + - name: Push changes + # && github.repository == 'envoyproxy/envoy' + if: ${{ needs.load.outputs.trusted }} + shell: bash + run: | + git -C mobile-docs status + exit 0 + git -C mobile-docs push origin master + timeout-minutes: 20 + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + upload-name: docs + upload-path: mobile/generated/docs - - uses: actions/upload-artifact@v3 - with: - name: docs - path: mobile/generated/docs + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-docs }} + needs: + - load + - docs + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-format.yml b/.github/workflows/mobile-format.yml index 13bf1b2dbb20..f661761eef1c 100644 --- a/.github/workflows/mobile-format.yml +++ b/.github/workflows/mobile-format.yml @@ -1,130 +1,158 @@ -name: mobile_format +name: Mobile/Format permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true -jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml - permissions: - contents: read - pull-requests: read - formatall: - if: ${{ needs.env.outputs.mobile_formatting == 'true' }} - needs: env +jobs: + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read packages: read - name: format_all - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 45 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Run formatters' - run: | - cd mobile - ./tools/check_format.sh + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-format - precommit: - if: ${{ needs.env.outputs.mobile_formatting == 'true' }} - needs: env + container: permissions: contents: read packages: read - name: precommit - runs-on: macos-12 - timeout-minutes: 45 - steps: - - uses: actions/checkout@v4 - - name: 'Install precommit' - run: brew install pre-commit - - name: 'Run precommit' - run: | - cd mobile - find mobile/* | pre-commit run --files + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-format }} + needs: load + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: ${{ matrix.args }} + command: ${{ matrix.command }} + container: ${{ matrix.container }} + request: ${{ needs.load.outputs.request }} + target: ${{ matrix.target }} + timeout-minutes: ${{ matrix.timeout-minutes }} + strategy: + fail-fast: false + matrix: + include: + - name: Format all + args: check + command: ./tools/check_format.sh + target: format-all + timeout-minutes: 45 + - name: Swift lint + args: >- + lint + --strict + command: swiftlint + container: >- + ghcr.io/realm/swiftlint:0.50.3 + target: swift-lint + timeout-minutes: 45 - swiftlint: - if: ${{ needs.env.outputs.mobile_formatting == 'true' }} - needs: env + host: permissions: contents: read packages: read - name: swift_lint - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 5 - container: - image: ghcr.io/realm/swiftlint:0.50.3 - steps: - - uses: actions/checkout@v4 - - name: 'Run Swift Lint (SwiftLint)' - run: swiftlint lint --strict + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-format }} + needs: load + uses: ./.github/workflows/_run.yml + with: + args: ${{ matrix.args }} + command: ${{ matrix.command }} + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: ${{ matrix.runs-on }} + source: ${{ matrix.source }} + steps-pre: ${{ matrix.steps-pre }} + steps-post: ${{ matrix.steps-post }} + target: ${{ matrix.target }} + timeout-minutes: ${{ matrix.timeout-minutes }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: Run drstring + args: >- + run + --config=remote-ci + @DrString//:drstring check + command: ./bazelw + runs-on: macos-12 + source: | + export DEVELOPER_DIR=/Applications/Xcode_14.1.app + target: drstring + timeout-minutes: 10 + - name: Pre-commit + command: pre-commit + args: >- + run + --files + mobile/* + source: | + pip install pre-commit + target: pre-commit + timeout-minutes: 45 + - name: Kotlin lint + args: >- + build + --config=mobile-remote-ci-macos + //library/kotlin/io/envoyproxy/envoymobile:envoy_lib_lint + //examples/kotlin/hello_world:hello_envoy_kt_lint + command: ./bazelw + runs-on: macos-12 + source: | + export DEVELOPER_DIR=/Applications/Xcode_14.1.app + ./ci/mac_ci_setup.sh + steps-pre: | + - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 + with: + java-version: '8' + java-package: jdk + architecture: x64 + distribution: zulu + steps-post: | + - name: Run Kotlin Formatter (ktlint) + shell: bash + run: | + cd mobile + ./bazelw build --config=remote-ci kotlin_format + target: kotlin-lint + timeout-minutes: 10 - drstring: - if: ${{ needs.env.outputs.mobile_formatting == 'true' }} - needs: env - permissions: - contents: read - packages: read - name: drstring - runs-on: macos-12 - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - - name: 'Run DrString' - env: - DEVELOPER_DIR: /Applications/Xcode_14.1.app - run: | - cd mobile - ./bazelw run --config=remote-ci @DrString//:drstring check - - kotlinlint: - if: ${{ needs.env.outputs.mobile_formatting == 'true' }} - needs: env + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read - packages: read - name: kotlin_lint - runs-on: macos-12 - timeout-minutes: 45 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 - with: - java-version: '8' - java-package: jdk - architecture: x64 - distribution: zulu - - run: | - cd mobile - ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Run Kotlin Lint (Detekt)' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-ci-macos \ - //library/kotlin/io/envoyproxy/envoymobile:envoy_lib_lint \ - //examples/kotlin/hello_world:hello_envoy_kt_lint - - name: 'Run Kotlin Formatter (ktlint)' - run: | - cd mobile - ./bazelw build --config=remote-ci kotlin_format + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-format }} + needs: + - load + - container + - host + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-ios_build.yml b/.github/workflows/mobile-ios_build.yml index 3852b2006c78..79763f7aba57 100644 --- a/.github/workflows/mobile-ios_build.yml +++ b/.github/workflows/mobile-ios_build.yml @@ -1,310 +1,206 @@ -name: ios_build +name: Mobile/iOS build permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed -concurrency: - group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} - cancel-in-progress: true jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + cache-docker: false + check-name: mobile-ios - iosbuild: - if: ${{ needs.env.outputs.mobile_ios_build == 'true' }} - needs: env + build: permissions: contents: read packages: read - name: ios_build - runs-on: macos-12 - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-ios }} + needs: load + name: ios-build + with: + args: ${{ matrix.args }} + command: ./bazelw + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: macos-12 + source: | ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build Envoy.framework distributable' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile ./bazelw shutdown - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ + steps-post: ${{ matrix.steps-post }} + target: ${{ matrix.target }} + timeout-minutes: ${{ matrix.timeout-minutes }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: Build Envoy.framework distributable + args: >- + build + --config=ios + --config=mobile-remote-ci-macos //library/swift:ios_framework + target: ios + timeout-minutes: 120 - swifthelloworld: - if: ${{ needs.env.outputs.mobile_ios_build == 'true' }} - name: swift_helloworld - needs: - - env - - iosbuild + hello-world: permissions: contents: read packages: read - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-ios }} + needs: + - load + # - build + name: ios-hello-world + with: + args: ${{ matrix.args }} + command: ./bazelw + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: macos-12 + source: | ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ + ./bazelw shutdown + steps-post: ${{ matrix.steps-post }} + target: ${{ matrix.target }} + timeout-minutes: ${{ matrix.timeout-minutes }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: Build swift hello world + args: >- + build + --config=ios + --config=mobile-remote-ci-macos //examples/swift/hello_world:app - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start simulator' - with: - timeout_minutes: 5 - max_attempts: 3 - command: ./mobile/ci/start_ios_simulator.sh - # Run the app in the background and redirect logs. - - name: 'Run app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw run \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //examples/swift/hello_world:app &> /tmp/envoy.log & - - run: | - sed '/received headers with status 200/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - name: 'Check connectivity' - - run: cat /tmp/envoy.log - if: ${{ failure() || cancelled() }} - name: 'Log app run' + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/ios/post@actions-v0.2.8 + with: + app: //examples/swift/hello_world:app + status: 200 + target: swift-hello-world + timeout-minutes: 50 - swiftbaselineapp: - if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} - needs: - - env - - iosbuild + apps: permissions: contents: read packages: read - name: swift_baseline_app - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile - ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //test/swift/apps/baseline:app - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start simulator' - with: - timeout_minutes: 5 - max_attempts: 3 - command: ./mobile/ci/start_ios_simulator.sh - # Run the app in the background and redirect logs. - - name: 'Run app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw run \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //test/swift/apps/baseline:app &> /tmp/envoy.log & - - run: | - sed '/received headers with status 301/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - name: 'Check connectivity' - - run: cat /tmp/envoy.log - if: ${{ failure() || cancelled() }} - name: 'Log app run' - - swiftexperimentalapp: - if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-ios-all }} needs: - - env - - iosbuild - permissions: - contents: read - packages: read - name: swift_experimental_app - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile + - load + - build + name: ios-apps + with: + args: ${{ matrix.args }} + command: ./bazelw + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: macos-12 + source: | ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ - --define=admin_functionality=enabled \ + steps-post: ${{ matrix.steps-post }} + target: ${{ matrix.target }} + timeout-minutes: 50 + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: Build swift baseline app + args: >- + --config=ios + --config=mobile-remote-ci-macos + //test/swift/apps/baseline:app + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/ios/post@actions-v0.2.8 + with: + app: //test/swift/apps/baseline:app + status: 200 + target: swift-baseline-app + - name: Build swift experimental app + args: >- + build + --config=ios + --config=mobile-remote-ci-macos + --define=admin_functionality=enabled //test/swift/apps/experimental:app - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start simulator' - with: - timeout_minutes: 5 - max_attempts: 3 - command: ./mobile/ci/start_ios_simulator.sh - # Run the app in the background and redirect logs. - - name: 'Run app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw run \ - --config=ios \ - --config=mobile-remote-ci-macos \ - --define=admin_functionality=enabled \ - //test/swift/apps/experimental:app &> /tmp/envoy.log & - - run: | - sed '/received headers with status 200/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - name: 'Check connectivity' - - run: cat /tmp/envoy.log - if: ${{ failure() || cancelled() }} - name: 'Log app run' - - swiftasyncawait: - if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} - needs: - - env - - iosbuild - permissions: - contents: read - packages: read - name: swift_async_await - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile - ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/ios/post@actions-v0.2.8 + with: + app: //test/swift/apps/experimental:app + args: >- + --config=ios + --config=mobile-remote-ci-macos + --define=admin_functionality=enabled + status: 200 + target: swift-experimental-app + - name: Build swift async await + args: >- + build + --config=ios + --config=mobile-remote-ci-macos //examples/swift/async_await:app - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start simulator' - with: - timeout_minutes: 5 - max_attempts: 3 - command: ./mobile/ci/start_ios_simulator.sh - # Run the app in the background and redirect logs. - - name: 'Run app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw run \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //examples/swift/async_await:app &> /tmp/envoy.log & - - run: | - checklogs () { - sed '/\[2\] Uploaded 7 MB of data/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - } - export -f checklogs - # TODO(phlax): figure if this needs this long - timeout 5m bash -c checklogs || { - retcode=$? - if [[ "$retcode" != 124 ]]; then - echo "Command failed" >&2 - elif grep -q "Upload failed" /tmp/envoy.log; then - echo "Upload failed" >&2 - else - echo "Upload timed out" >&2 - fi - exit 1 - } - if: steps.should_run.outputs.run_ci_job == 'true' - name: 'Check upload succeeded' - - run: cat /tmp/envoy.log - if: ${{ failure() || cancelled() }} - name: 'Log app run' + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/ios/post@actions-v0.2.8 + with: + app: //examples/swift/async_await:app + target: swift-async-await + - name: Build objc hello world + args: >- + --config=ios + --config=mobile-remote-ci-macos + //examples/objective-c/hello_world:app + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/ios/post@actions-v0.2.8 + with: + app: //examples/objective-c/hello_world:app + status: 301 + target: objc-hello-world - objchelloworld: - if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} - needs: - - env - - iosbuild + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read - packages: read - name: objc_helloworld - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile - ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //examples/objective-c/hello_world:app - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start simulator' - with: - timeout_minutes: 5 - max_attempts: 3 - command: ./mobile/ci/start_ios_simulator.sh - # Run the app in the background and redirect logs. - - name: 'Run app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw run \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //examples/objective-c/hello_world:app &> /tmp/envoy.log & - - run: sed '/received headers with status 301/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - name: 'Check connectivity' - - run: cat /tmp/envoy.log - if: ${{ failure() || cancelled() }} - name: 'Log app run' + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-ios }} + needs: + - load + - build + - hello-world + - apps + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-ios_tests.yml b/.github/workflows/mobile-ios_tests.yml index a0181f19556e..1e6a6a674a64 100644 --- a/.github/workflows/mobile-ios_tests.yml +++ b/.github/workflows/mobile-ios_tests.yml @@ -1,76 +1,95 @@ -name: ios_tests +name: Mobile/iOS tests permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed -concurrency: - group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} - cancel-in-progress: true jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + cache-docker: false + check-name: mobile-ios-tests - swifttests: - if: ${{ needs.env.outputs.mobile_ios_tests == 'true' }} - needs: env + tests: permissions: contents: read packages: read - name: swift_tests - runs-on: macos-12 - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - name: 'Install dependencies' - run: | - cd mobile + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-ios-tests }} + needs: load + name: ios-tests + with: + args: ${{ matrix.args }} + command: ./bazelw + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: macos-12 + source: | ./ci/mac_ci_setup.sh - - name: 'Run swift library tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw test \ - --experimental_ui_max_stdouterr_bytes=10485760 \ - --config=ios \ - --build_tests_only \ - --config=mobile-remote-ci-macos \ + steps-post: ${{ matrix.steps-post }} + target: ${{ matrix.target }} + timeout-minutes: ${{ matrix.timeout-minutes }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: Run swift library tests + args: >- + test + --experimental_ui_max_stdouterr_bytes=10485760 + --config=ios + --build_tests_only + --config=mobile-remote-ci-macos //test/swift/... + target: swift-tests + timeout-minutes: 120 + - name: Run Objective-C library tests + args: >- + test + --config=ios + --build_tests_only + --config=mobile-remote-ci-macos + //test/objective-c/... + //test/cc/unit:envoy_config_test + target: c-and-objc-tests + timeout-minutes: 120 - objctests: - if: ${{ needs.env.outputs.mobile_ios_tests == 'true' }} - needs: env + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read - packages: read - name: c_and_objc_tests - runs-on: macos-12 - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - name: 'Install dependencies' - run: | - cd mobile - ./ci/mac_ci_setup.sh - - name: 'Run Objective-C library tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw test \ - --config=ios \ - --build_tests_only \ - --config=mobile-remote-ci-macos \ - //test/objective-c/... \ - //test/cc/unit:envoy_config_test + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-ios-tests }} + needs: + - load + - tests + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-perf.yml b/.github/workflows/mobile-perf.yml index c7c34e9ebab3..7a0e43573d52 100644 --- a/.github/workflows/mobile-perf.yml +++ b/.github/workflows/mobile-perf.yml @@ -1,115 +1,128 @@ -name: mobile_perf +name: Mobile/Perf permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - sizecurrent: - if: ${{ github.repository == 'envoyproxy/envoy' }} + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read packages: read - name: size_current - runs-on: ubuntu-22.04 - timeout-minutes: 120 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - env: - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - steps: - - uses: actions/checkout@v4 - - name: Ensure files don't leak back into the main binary - run: rm source/common/listener_manager/listener_manager_impl.h source/server/overload_manager_impl.cc source/common/network/listen_socket_impl.h source/common/network/tcp_listener_impl.h source/server/guarddog_impl.h source/server/watchdog_impl.h source/server/options_impl.cc - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Build test binary' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-release-clang \ - //test/performance:test_binary_size - - uses: actions/upload-artifact@v3 - with: - name: sizecurrent - path: mobile/bazel-bin/test/performance/test_binary_size + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-perf - sizemain: - if: ${{ github.repository == 'envoyproxy/envoy' }} + build: permissions: contents: read packages: read - name: size_main - runs-on: ubuntu-22.04 - timeout-minutes: 90 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - env: - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - steps: - - uses: actions/checkout@v4 - with: - ref: main - - name: Add safe directory - run: | - git config --global --add safe.directory /__w/envoy/envoy - - name: 'Build test binary' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-release-clang \ + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-perf }} + needs: load + name: Build test binaries + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: ${{ matrix.args }} + command: ./bazelw + ref: ${{ matrix.ref }} + request: ${{ needs.load.outputs.request }} + source: ${{ matrix.source }} + container-output: | + "bazel-bin/test/performance/test_binary_size": /tmp/mobile/bazel-bin/test/performance/test_binary_size/${{ matrix.target }} + target: ${{ matrix.target }} + upload-name: ${{ matrix.target }} + upload-path: /tmp/mobile/bazel-bin/test/performance/test_binary_size + timeout-minutes: 90 + strategy: + matrix: + include: + - name: Current size + args: >- + build + --config=mobile-remote-release-clang + //test/performance:test_binary_size + # Ensure files don't leak back into the main binary + source: >- + rm + source/common/listener_manager/listener_manager_impl.h + source/server/overload_manager_impl.cc + source/common/network/listen_socket_impl.h + source/common/network/tcp_listener_impl.h + source/server/guarddog_impl.h + source/server/watchdog_impl.h + source/server/options_impl.cc + target: size-current + - name: Main size + args: >- + build + --config=mobile-remote-release-clang //test/performance:test_binary_size - - uses: actions/upload-artifact@v3 - with: - name: sizemain - path: mobile/bazel-bin/test/performance/test_binary_size + ref: main + target: size-main - sizecompare: - if: ${{ github.repository == 'envoyproxy/envoy' }} - needs: - - sizecurrent - - sizemain + compare: permissions: contents: read packages: read + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-perf }} + needs: + - load + - build name: size_compare - runs-on: ubuntu-22.04 - timeout-minutes: 30 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 - with: - name: sizecurrent - path: dist/sizecurrent - - uses: actions/download-artifact@v3 - with: - name: sizemain - path: dist/sizemain - - name: 'Strip and Zip binary' - run: | - ls -lh dist/ - strip -s -o dist/main.stripped dist/sizemain/test_binary_size - strip -s -o dist/current.stripped dist/sizecurrent/test_binary_size + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: >- + ../dist/main.zip + ../dist/current.zip + command: ./ci/test_size_regression.sh + downloads: | + size-current: dist/sizecurrent + size-main: dist/sizemain + request: ${{ needs.load.outputs.request }} + runs-on: ubuntu-22.04 + source: | + strip -s -o dist/main.stripped dist/sizemain/size-main + strip -s -o dist/current.stripped dist/sizecurrent/size-current zip -9 dist/main.zip dist/main.stripped zip -9 dist/current.zip dist/current.stripped - - name: 'Test size regression' - run: | - cd mobile - ./ci/test_size_regression.sh ../dist/main.zip ../dist/current.zip + target: size-compare + timeout-minutes: 30 + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-perf }} + needs: + - load + - build + - compare + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-release.yml b/.github/workflows/mobile-release.yml index 8b3d7e08648f..05c093364cfa 100644 --- a/.github/workflows/mobile-release.yml +++ b/.github/workflows/mobile-release.yml @@ -12,7 +12,7 @@ on: jobs: env: if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + uses: ./.github/workflows/_load.yml permissions: contents: read pull-requests: read @@ -29,10 +29,10 @@ jobs: contents: read packages: read name: android_release_artifacts - runs-on: ${{ needs.env.outputs.agent_ubuntu }} + runs-on: ${{ fromJSON(needs.request.outputs.request).ci.agent-ubuntu }} timeout-minutes: 120 container: - image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} + image: ${{ needs.env.outputs.build-image-mobile }} env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ @@ -145,10 +145,10 @@ jobs: contents: read packages: read name: android_xds_release_artifacts - runs-on: ${{ needs.env.outputs.agent_ubuntu }} + runs-on: ${{ fromJSON(needs.env.outputs.request).ci.agent-ubuntu }} timeout-minutes: 120 container: - image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} + image: ${{ needs.env.outputs.build-image-mobile }} env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/.github/workflows/mobile-release_validation.yml b/.github/workflows/mobile-release_validation.yml index 76775184fddb..70e0a212a91c 100644 --- a/.github/workflows/mobile-release_validation.yml +++ b/.github/workflows/mobile-release_validation.yml @@ -1,59 +1,93 @@ -name: mobile_release_validation +name: Mobile/Release validation permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + cache-docker: false + check-name: mobile-release-validation - validate_swiftpm_example: - if: ${{ needs.env.outputs.mobile_release_validation == 'true' }} - needs: env + validate-swiftpm-example: permissions: contents: read packages: read - name: validate_swiftpm_example - runs-on: macos-12 - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-release-validation }} + needs: load + uses: ./.github/workflows/_run.yml + name: Build xframework + with: + args: >- + build + --config=ios + --config=mobile-remote-ci-macos + //:ios_xcframework + command: ./bazelw + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: macos-12 + source: | ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build xcframework' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //:ios_xcframework - # Ignore errors: Bad CRC when unzipping large files: https://bbs.archlinux.org/viewtopic.php?id=153011 - - run: | - unzip mobile/bazel-bin/library/swift/Envoy.xcframework.zip \ - -d mobile/examples/swift/swiftpm/Packages \ - || : - name: 'Unzip xcframework' - - run: | - xcodebuild -project mobile/examples/swift/swiftpm/EnvoySwiftPMExample.xcodeproj \ - -scheme EnvoySwiftPMExample \ - -destination platform="iOS Simulator,name=iPhone 14 Pro Max,OS=16.1" - name: 'Build app' - # TODO(jpsim): Run app and inspect logs to validate + # Ignore errors: Bad CRC when unzipping large files: https://bbs.archlinux.org/viewtopic.php?id=153011 + steps-post: | + - run: | + unzip mobile/bazel-bin/library/swift/Envoy.xcframework.zip \ + -d mobile/examples/swift/swiftpm/Packages \ + || : + shell: bash + name: Unzip xcframework + - run: | + xcodebuild -project mobile/examples/swift/swiftpm/EnvoySwiftPMExample.xcodeproj \ + -scheme EnvoySwiftPMExample \ + -destination platform="iOS Simulator,name=iPhone 14 Pro Max,OS=16.1" + shell: bash + name: Build app + # TODO(jpsim): Run app and inspect logs to validate + target: validate-swiftpm-example + timeout-minutes: 120 + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-release-validation }} + needs: + - load + - validate-swiftpm-example + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-tsan.yml b/.github/workflows/mobile-tsan.yml index 89bd61dd48eb..ef9effbba871 100644 --- a/.github/workflows/mobile-tsan.yml +++ b/.github/workflows/mobile-tsan.yml @@ -1,51 +1,71 @@ -name: mobile_tsan +name: Mobile/TSAN permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read + packages: read pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-tsan + run-id: ${{ github.event.workflow_run.id }} tsan: - if: ${{ needs.env.outputs.mobile_tsan == 'true' }} - needs: env permissions: contents: read packages: read + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-tsan }} + needs: load name: tsan - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 90 - container: - image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} - env: - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Run tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw test \ - --test_env=ENVOY_IP_TEST_VERSIONS=v4only \ - --config=mobile-remote-ci-linux-tsan \ - //test/common/... \ - //test/cc/... + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: >- + test + --test_env=ENVOY_IP_TEST_VERSIONS=v4only + --config=mobile-remote-ci-linux-tsan + //test/common/... + //test/cc/... + request: ${{ needs.load.outputs.request }} + target: tsan + timeout-minutes: 90 + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-tsan }} + needs: + - load + - tsan + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/request.yml b/.github/workflows/request.yml new file mode 100644 index 000000000000..a245052db14a --- /dev/null +++ b/.github/workflows/request.yml @@ -0,0 +1,39 @@ +# This file must live on every branch and pass necessary secrets and permissions +# to initiate the request +name: Request + +permissions: + contents: read + +on: + pull_request_target: + push: + branches: + - main + - release/v* + +concurrency: + group: | + ${{ github.head_ref + || github.run_id + }}-${{ github.workflow }}-request + cancel-in-progress: true + + +jobs: + request: + # For branches this can be pinned to a specific version if required + # NB: `uses` cannot be dynamic so it _must_ be hardcoded anywhere it is read + uses: envoyproxy/envoy/.github/workflows/_request.yml@main + if: ${{ vars.ENVOY_CI || github.repository == 'envoyproxy/envoy' }} + permissions: + actions: read + contents: read + # required for engflow/bazel caching (not yet used) + packages: read + # required to fetch merge commit + pull-requests: read + secrets: + # these are required to start checks + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} diff --git a/.github/workflows/workflow-complete.yml b/.github/workflows/workflow-complete.yml deleted file mode 100644 index 7d7d5a005a0a..000000000000 --- a/.github/workflows/workflow-complete.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Workflow complete -# This workflow is only required for externally triggered jobs that have manually -# set the check status for a commit/PR - -permissions: - contents: read - -on: - # Do not run untrusted code here - workflow_run: - workflows: - - Publish & verify - types: - - completed - -jobs: - complete: - if: ${{ github.actor == 'trigger-workflow-envoy[bot]' }} - runs-on: ubuntu-22.04 - permissions: - contents: read - statuses: write - steps: - - name: 'Download artifact' - uses: actions/github-script@v6 - with: - script: | - let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.payload.workflow_run.id, - }); - let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { - return artifact.name == "state_sha" - })[0]; - let download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - let fs = require('fs'); - fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/state_sha.zip`, Buffer.from(download.data)); - - - run: | - set -e - unzip state_sha.zip - STATE_SHA="$(cat state_sha)" - echo "state_sha=$STATE_SHA" >> "$GITHUB_OUTPUT" - STATE="${{ github.event.workflow_run.conclusion }}" - if [[ ${STATE} != "success" ]]; then - STATE=failure - fi - echo "state=${STATE}" >> "$GITHUB_OUTPUT" - id: job - - name: Complete status check - uses: envoyproxy/toolshed/gh-actions/status@actions-v0.1.8 - with: - authToken: ${{ secrets.GITHUB_TOKEN }} - context: Verify/examples - state: ${{ steps.job.outputs.state }} - sha: ${{ steps.job.outputs.state_sha }} - target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }} diff --git a/BUILD b/BUILD index 2418b2b331ae..e7f90fcd9ea7 100644 --- a/BUILD +++ b/BUILD @@ -15,7 +15,7 @@ exports_files([ ".coveragerc", "CODEOWNERS", "OWNERS.md", - ".github/workflows/_env.yml", + ".github/config.yml", ]) alias( diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 8e4e0b6d2e54..094572f3b56d 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -51,6 +51,7 @@ else BUILD_DIR_MOUNT_DEST=/build SOURCE_DIR="${PWD}" SOURCE_DIR_MOUNT_DEST=/source + ENVOY_DOCKER_SOURCE_DIR="${ENVOY_DOCKER_SOURCE_DIR:-${SOURCE_DIR_MOUNT_DEST}}" START_COMMAND=( "/bin/bash" "-lc" @@ -59,7 +60,7 @@ else && usermod -a -G pcap envoybuild \ && chown envoybuild:envoygroup /build \ && chown envoybuild /proc/self/fd/2 \ - && sudo -EHs -u envoybuild bash -c 'cd /source && $*'") + && sudo -EHs -u envoybuild bash -c 'cd ${ENVOY_DOCKER_SOURCE_DIR} && $*'") fi if [[ -n "$ENVOY_DOCKER_PLATFORM" ]]; then diff --git a/mobile/ci/mac_ci_setup.sh b/mobile/ci/mac_ci_setup.sh index 41c5e3c26873..4a6016372016 100755 --- a/mobile/ci/mac_ci_setup.sh +++ b/mobile/ci/mac_ci_setup.sh @@ -34,7 +34,7 @@ function is_installed { function install { echo "Installing $1" - if ! retry brew install "$1"; then + if ! retry brew install --quiet "$1"; then echo "Failed to install $1" exit 1 fi @@ -67,5 +67,6 @@ if [[ "${1:-}" == "--android" ]]; then # Download and set up build-tools 30.0.3, 31.0.0 is missing dx.jar. $SDKMANAGER --install "build-tools;30.0.3" - echo "ANDROID_NDK_HOME=${ANDROID_HOME}/ndk/21.4.7075529" >> "$GITHUB_ENV" + ANDROID_NDK_HOME="${ANDROID_HOME}/ndk/21.4.7075529" + export ANDROID_NDK_HOME fi diff --git a/mobile/ci/test_size_regression.sh b/mobile/ci/test_size_regression.sh index d5b03618a8a2..c80afb73c10a 100755 --- a/mobile/ci/test_size_regression.sh +++ b/mobile/ci/test_size_regression.sh @@ -1,4 +1,6 @@ -#!/bin/bash +#!/bin/bash -e + +set -o pipefail # Checks the absolute size and the relative size increase of a file. diff --git a/mobile/tools/what_to_run.sh b/mobile/tools/what_to_run.sh deleted file mode 100755 index e50ac44bbd85..000000000000 --- a/mobile/tools/what_to_run.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -BRANCH_NAME="$GITHUB_REF_NAME" -BASE_COMMIT="$(git merge-base origin/main HEAD)" -CHANGED_FILES="$(git diff "${BASE_COMMIT}" --name-only)" -CHANGE_MATCH='^mobile/|^bazel/repository_locations\.bzl|^\.bazelrc|^\.bazelversion|^\.github/workflows/mobile-*|^\.github/workflows/_env.yml|^tools/code_format/check_format.py|bazel/external/quiche.BUILD' - -# The logic in this file is roughly: -# -# pull_request + changed files = run all mobile CI -# -# main = run some mobile CI -# -# all other commits = run minimal mobile CI (these jobs have no conditions) -# -# Branches are not currently tested, altho that should be restricted by the workflow - - -run_default_ci () { - { - echo "mobile_android_build=true" - echo "mobile_android_tests=true" - echo "mobile_asan=true" - echo "mobile_cc_tests=true" - echo "mobile_compile_time_options=true" - echo "mobile_coverage=true" - echo "mobile_formatting=true" - echo "mobile_ios_build=true" - echo "mobile_ios_tests=true" - echo "mobile_release_validation=true" - echo "mobile_tsan=true" - } >> "$GITHUB_OUTPUT" -} - -run_ci_for_changed_files () { - run_default_ci - { - echo "mobile_android_build_all=true" - echo "mobile_ios_build_all=true" - } >> "$GITHUB_OUTPUT" -} - -if [[ "$BRANCH_NAME" == "main" ]]; then - run_default_ci - exit 0 -fi - -if grep -qE "$CHANGE_MATCH" <<< "$CHANGED_FILES"; then - run_ci_for_changed_files -fi diff --git a/tools/dependency/BUILD b/tools/dependency/BUILD index a7fb7edd96f5..ff0467298b73 100644 --- a/tools/dependency/BUILD +++ b/tools/dependency/BUILD @@ -95,13 +95,9 @@ genrule( envoy_genjson( name = "build-images", filter = """ - .[0].on.workflow_call.inputs - | to_entries - | map(select(.key | startswith("build_image") and . != "build_image_repo") - | {(.key | gsub("build_image_"; "")): .value.default}) - | add + .[0]["build-image"] """, - yaml_srcs = ["//:.github/workflows/_env.yml"], + yaml_srcs = ["//:.github/config.yml"], ) sh_binary( diff --git a/tools/dependency/version.sh b/tools/dependency/version.sh index a4274601790d..2309c93d82f0 100755 --- a/tools/dependency/version.sh +++ b/tools/dependency/version.sh @@ -11,4 +11,4 @@ if [[ -z "$DEP" ]]; then exit 1 fi -$JQ -r ".${DEP}" "$VERSIONS" +$JQ -r ".[\"${DEP}\"]" "$VERSIONS"