diff --git a/.azure-pipelines/stage/prechecks.yml b/.azure-pipelines/stage/prechecks.yml index c4e2d779a6ba1..04bb832c8f962 100644 --- a/.azure-pipelines/stage/prechecks.yml +++ b/.azure-pipelines/stage/prechecks.yml @@ -15,6 +15,15 @@ parameters: - name: authGCP type: string default: "" +- name: authGithubWorkflow + type: string + default: "" +- name: authGithubWorkflowAppId + type: string + default: "" +- name: authGithubWorkflowInstallId + type: string + default: "" - name: authGPGPassphrase type: string default: "" @@ -34,6 +43,11 @@ parameters: type: string default: true +- name: publishGithubRelease + displayName: "Publish Github release" + type: string + default: false + # Timeout/s - name: timeoutPrechecks type: number @@ -219,10 +233,51 @@ jobs: # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage condition: | and( - eq(variables['Build.Reason'], 'PullRequest'), in(dependencies.prechecks.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), + in(dependencies.precheck_release_x64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), in(dependencies.precheck_release_arm64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) steps: - - checkout: none + - task: DownloadSecureFile@1 + name: WorkflowTriggerKey + displayName: 'Download workflow trigger key' + inputs: + secureFile: '${{ parameters.authGithubWorkflow }}' - bash: | - echo "Prechecked" + set -e + KEY="$(cat $(WorkflowTriggerKey.secureFilePath) | base64 -w0)" + echo "##vso[task.setvariable variable=value;isoutput=true]$KEY" + name: key + - template: ../ci.yml + parameters: + ciTarget: verify.trigger + cacheName: verify-trigger + authGithub: "$(key.value)" + cacheVersion: $(cacheKeyBazel) + publishEnvoy: false + publishTestResults: false + env: + ENVOY_REPO: $(Build.Repository.Name) + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + ENVOY_HEAD_REF: "$(Build.SourceBranch)" + ENVOY_BRANCH: "$(System.PullRequest.TargetBranch)" + ENVOY_COMMIT: "$(System.PullRequest.SourceCommitId)" + ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + ENVOY_HEAD_REF: "$(Build.SourceBranchName)" + ENVOY_BRANCH: "$(Build.SourceBranch)" + # github auth + GITHUB_APP_ID: ${{ parameters.authGithubWorkflowAppId }} + GITHUB_INSTALL_ID: ${{ parameters.authGithubWorkflowInstallId }} + # rbe env + GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} + stepsPre: + - bash: | + set -e + if [[ "$BUILD_REASON" == "PullRequest" ]]; then + DOWNLOAD_PATH="$(git rev-parse HEAD | head -c7)" + else + DOWNLOAD_PATH="${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-${BUILD_SOURCEBRANCHNAME}}" + fi + curl -sLO "https://storage.googleapis.com/${{ parameters.bucketGCP }}/${DOWNLOAD_PATH}/release/release.signed.tar.zst" + mkdir -p $(Build.StagingDirectory)/release.signed + mv release.signed.tar.zst $(Build.StagingDirectory)/release.signed + displayName: Fetch signed release diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml deleted file mode 100644 index dc1f811cdf63d..0000000000000 --- a/.azure-pipelines/stage/publish.yml +++ /dev/null @@ -1,376 +0,0 @@ -parameters: - -- name: bucketGCP - type: string - default: "" - -## Timeouts -# Workaround for https://github.com/envoyproxy/envoy/issues/26634 -- name: timeoutDockerPublish - displayName: "Timout Docker publish" - type: number - # in minutes - default: 18 -- name: timeoutDockerBuild - displayName: "Timout Docker build" - type: number - default: 500 - -# Auth -- name: authGCP - type: string - default: "" -- name: authGithubWorkflow - type: string - default: "" -- name: authGithubWorkflowAppId - type: string - default: "" -- name: authGithubWorkflowInstallId - type: string - default: "" -- name: authGPGPassphrase - type: string - default: "" -- name: authGPGPath - type: string - default: "" -- name: authGPGKey - type: string - default: "" -- name: authDockerUser - type: string - default: "" -- name: authDockerPassword - type: string - default: "" - -- name: bazelConfigRBE - type: string - default: --config=remote-ci --config=rbe-google --jobs=$(RbeJobs) - -- name: runBuild - displayName: "Run Build" - type: string - default: true -- name: runDocker - displayName: "Run Docker" - type: string - default: true -- name: runPackaging - displayName: "Run Packaging" - type: string - default: true - -- name: publishDockerhub - displayName: "Publish Dockerhub" - type: string - default: false -- name: publishGithubRelease - displayName: "Publish Github release" - type: string - default: false - -jobs: - -- job: release_x64 - displayName: Binary release - condition: | - and(not(canceled()), - eq(${{ parameters.runBuild }}, 'true')) - timeoutInMinutes: 120 - pool: - vmImage: $(agentUbuntu) - steps: - - template: ../ci.yml - parameters: - artifactName: release - ciTarget: release.server_only - cacheName: release-server-only - bazelConfigRBE: ${{ parameters.bazelConfigRBE }} - cacheVersion: $(cacheKeyBazel) - cacheTestResults: false - publishTestResults: false - rbe: true - -- job: release_arm64 - displayName: Binary release (arm64) - condition: | - and(not(canceled()), - eq(${{ parameters.runBuild }}, 'true')) - timeoutInMinutes: 180 - pool: envoy-arm-large - steps: - - bash: | - CI_TARGET="release.server_only" - echo "${CI_TARGET}" - echo "##vso[task.setvariable variable=value;isoutput=true]${CI_TARGET}" - name: target - - template: ../ci.yml - parameters: - artifactName: release - ciTarget: release.server_only - cacheName: release-server-only - bazelConfigRBE: ${{ parameters.bazelConfigRBE }} - bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" - cacheVersion: $(cacheKeyBazel) - cacheTestResults: false - artifactSuffix: .arm64 - publishTestResults: false - rbe: false - -- job: docker - displayName: "Docker (Linux multi arch)" - dependsOn: - - release_x64 - - release_arm64 - condition: | - and(not(canceled()), - eq(${{ parameters.runDocker }}, 'true')) - timeoutInMinutes: 120 - pool: - vmImage: $(agentUbuntu) - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "release" - itemPattern: "release/**/bin/*" - targetPath: $(Build.StagingDirectory) - - template: ../ci.yml - parameters: - ciTarget: docker-upload - cacheName: docker-upload - publishEnvoy: false - publishTestResults: false - pathDockerBind: "" - tmpfsCacheDisabled: true - diskspaceHack: true - env: - GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} - stepsPre: - - bash: | - echo "disk space at beginning of Docker build:" - df -h - displayName: "Check disk space before Docker build" - # TODO(phlax): switch docker <> docker-upload as main task - - bash: | - set -e - mkdir -p $(Build.StagingDirectory)/envoy - rm -rf $(Build.StagingDirectory)/envoy/* - mv $(Build.StagingDirectory)/release/* $(Build.StagingDirectory)/envoy - ./ci/run_envoy_docker.sh 'ci/do_ci.sh docker' - displayName: Build Docker images - timeoutInMinutes: ${{ parameters.timeoutDockerPublish }} - workingDirectory: $(Build.SourcesDirectory) - env: - CI_BRANCH: $(Build.SourceBranch) - CI_SHA1: $(Build.SourceVersion) - DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }} - DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }} - DOCKER_BUILD_TIMEOUT: ${{ parameters.timeoutDockerBuild }} - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - ENVOY_DOCKER_IN_DOCKER: 1 - - stepsPost: - - script: | - ci/run_envoy_docker.sh 'ci/do_ci.sh dockerhub-publish' - condition: | - and(not(canceled()), succeeded(), - eq(${{ parameters.publishDockerhub }}, 'true')) - displayName: "Publish Dockerhub description and README" - env: - GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} - DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }} - DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }} - -- job: package_x64 - displayName: Linux debs (x64) - dependsOn: ["release_x64"] - condition: | - and(not(canceled()), - eq(${{ parameters.runPackaging }}, 'true')) - timeoutInMinutes: 120 - pool: - vmImage: $(agentUbuntu) - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "release" - itemPattern: "release/x64/bin/*" - targetPath: $(Build.StagingDirectory) - - template: ../ci.yml - parameters: - ciTarget: distribution - cacheName: distribution - publishTestResults: false - stepsPre: - - template: ../gpg.yml - parameters: - authGPGPassphrase: ${{ parameters.authGPGPassphrase }} - authGPGPath: ${{ parameters.authGPGPath }} - authGPGKey: ${{ parameters.authGPGKey }} - pathGPGConfiguredHome: /build/.gnupg - pathGPGHome: $(Build.StagingDirectory)/.gnupg - stepsPost: - - bash: | - set -e - rm -rf $(Build.StagingDirectory)/.gnupg - -- job: package_arm64 - displayName: Linux debs (arm64) - dependsOn: ["release_arm64"] - condition: | - and(not(canceled()), - eq(${{ parameters.runPackaging }}, 'true')) - timeoutInMinutes: 120 - pool: "envoy-arm-small" - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "release" - itemPattern: "release/arm64/bin/*" - targetPath: $(Build.StagingDirectory) - - - template: ../ci.yml - parameters: - managedAgent: false - ciTarget: distribution - cacheName: distribution - rbe: false - artifactSuffix: ".arm64" - bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" - publishTestResults: false - tmpfsDockerDisabled: true - stepsPre: - - template: ../gpg.yml - parameters: - authGPGPassphrase: ${{ parameters.authGPGPassphrase }} - authGPGPath: ${{ parameters.authGPGPath }} - authGPGKey: ${{ parameters.authGPGKey }} - pathGPGConfiguredHome: /build/.gnupg - pathGPGHome: $(Build.StagingDirectory)/.gnupg - stepsPost: - - bash: | - set -e - rm -rf $(Build.StagingDirectory)/.gnupg - -- job: signed_release - displayName: Signed binaries - dependsOn: - - package_x64 - - package_arm64 - condition: | - and(succeeded(), - eq(${{ parameters.runPackaging }}, 'true')) - timeoutInMinutes: 120 - pool: - vmImage: $(agentUbuntu) - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "release" - itemPattern: "release/**/bin/*" - targetPath: $(Build.StagingDirectory) - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "distribution" - itemPattern: "distribution/**/packages.*.tar.gz" - targetPath: $(Build.StagingDirectory) - - template: ../ci.yml - parameters: - ciTarget: release.signed - cacheName: release-signed - publishTestResults: false - env: - GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} - stepsPre: - - template: ../gpg.yml - parameters: - authGPGPassphrase: ${{ parameters.authGPGPassphrase }} - authGPGPath: ${{ parameters.authGPGPath }} - authGPGKey: ${{ parameters.authGPGKey }} - pathGPGConfiguredHome: /build/.gnupg - pathGPGHome: $(Build.StagingDirectory)/.gnupg - - bash: | - set -e -o pipefail - mkdir -p distribution/custom - cp -a $(Build.StagingDirectory)/*/*64 distribution/custom/ - workingDirectory: $(Build.SourcesDirectory) - -- job: success - dependsOn: ["docker", "signed_release"] - displayName: Success (linux artefacts) - pool: - vmImage: $(agentUbuntu) - # This condition ensures that this (required) check passes if all of - # the preceding checks either pass or are skipped - # adapted from: - # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage - condition: | - and( - in(dependencies.docker.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), - in(dependencies.signed_release.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) - steps: - - checkout: none - - bash: | - echo "building publishable assets complete" - -- job: github - displayName: Publish release tag - dependsOn: ["success"] - condition: | - and(not(canceled()), - in(dependencies.success.result, 'Succeeded', 'SucceededWithIssues'), - eq(${{ parameters.publishGithubRelease }}, 'true')) - pool: - vmImage: $(agentUbuntu) - steps: - - task: DownloadSecureFile@1 - name: WorkflowTriggerKey - displayName: 'Download workflow trigger key' - inputs: - secureFile: '${{ parameters.authGithubWorkflow }}' - - bash: | - set -e - KEY="$(cat $(WorkflowTriggerKey.secureFilePath) | base64 -w0)" - echo "##vso[task.setvariable variable=value;isoutput=true]$KEY" - name: key - - template: ../ci.yml - parameters: - ciTarget: verify.trigger - cacheName: verify-trigger - authGithub: "$(key.value)" - cacheVersion: $(cacheKeyBazel) - publishEnvoy: false - publishTestResults: false - env: - ENVOY_REPO: $(Build.Repository.Name) - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - ENVOY_HEAD_REF: "$(Build.SourceBranch)" - ENVOY_BRANCH: "$(System.PullRequest.TargetBranch)" - ENVOY_COMMIT: "$(System.PullRequest.SourceCommitId)" - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - ENVOY_HEAD_REF: "$(Build.SourceBranchName)" - ENVOY_BRANCH: "$(Build.SourceBranch)" - # github auth - GITHUB_APP_ID: ${{ parameters.authGithubWorkflowAppId }} - GITHUB_INSTALL_ID: ${{ parameters.authGithubWorkflowInstallId }} - # rbe env - GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} - stepsPre: - - bash: | - set -e - if [[ "$BUILD_REASON" == "PullRequest" ]]; then - DOWNLOAD_PATH="$(git rev-parse HEAD | head -c7)" - else - DOWNLOAD_PATH="${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-${BUILD_SOURCEBRANCHNAME}}" - fi - curl -sLO "https://storage.googleapis.com/${{ parameters.bucketGCP }}/${DOWNLOAD_PATH}/release/release.signed.tar.zst" - mkdir -p $(Build.StagingDirectory)/release.signed - mv release.signed.tar.zst $(Build.StagingDirectory)/release.signed - displayName: Fetch signed release diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index 72438bf0ab462..8693671b524fe 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -47,16 +47,22 @@ stages: - stage: prechecks displayName: Prechecks dependsOn: ["env"] + variables: + PUBLISH_GITHUB_RELEASE: $[stageDependencies.env.repo.outputs['publish.githubRelease']] jobs: - template: stage/prechecks.yml parameters: concurrencyPrechecks: ${{ parameters.concurrencyPrechecks }} cacheTestResults: ${{ parameters.cacheTestResults }} authGCP: $(GcpServiceAccountKey) + authGithubWorkflow: $(GitHubPublicRepoWorkflowKey) + authGithubWorkflowAppId: $(GitHubPublicRepoWorkflowAppId) + authGithubWorkflowInstallId: $(GitHubPublicRepoWorkflowInstallId) authGPGPassphrase: $(MaintainerGPGKeyPassphrase) authGPGKey: $(MaintainerGPGKeySecureFileDownloadPath) authGPGPath: $(MaintainerGPGKey.secureFilePath) bucketGCP: $(GcsArtifactBucket) + publishGithubRelease: variables['PUBLISH_GITHUB_RELEASE'] runPrechecks: stageDependencies.env.repo.outputs['run.releaseTests'] - stage: check @@ -73,33 +79,3 @@ stages: authGCP: $(GcpServiceAccountKey) bucketGCP: $(GcsArtifactBucket) runChecks: variables['RUN_CHECKS'] - -- stage: publish - displayName: Publish - dependsOn: ["env", "prechecks"] - variables: - RUN_BUILD: $[stageDependencies.env.repo.outputs['run.build']] - RUN_DOCKER: $[stageDependencies.env.repo.outputs['run.docker']] - RUN_PACKAGING: $[stageDependencies.env.repo.outputs['run.packaging']] - PUBLISH_GITHUB_RELEASE: $[stageDependencies.env.repo.outputs['publish.githubRelease']] - PUBLISH_DOCKERHUB: $[stageDependencies.env.repo.outputs['publish.dockerhub']] - jobs: - - template: stage/publish.yml - parameters: - authDockerUser: $(DockerUsername) - authDockerPassword: $(DockerPassword) - authGCP: $(GcpServiceAccountKey) - authGithubWorkflow: $(GitHubPublicRepoWorkflowKey) - authGithubWorkflowAppId: $(GitHubPublicRepoWorkflowAppId) - authGithubWorkflowInstallId: $(GitHubPublicRepoWorkflowInstallId) - authGPGPassphrase: $(MaintainerGPGKeyPassphrase) - authGPGKey: $(MaintainerGPGKeySecureFileDownloadPath) - authGPGPath: $(MaintainerGPGKey.secureFilePath) - bucketGCP: $(GcsArtifactBucket) - timeoutDockerBuild: ${{ parameters.timeoutDockerBuild }} - timeoutDockerPublish: ${{ parameters.timeoutDockerPublish }} - runBuild: variables['RUN_BUILD'] - runDocker: variables['RUN_DOCKER'] - runPackaging: variables['RUN_PACKAGING'] - publishDockerhub: variables['PUBLISH_DOCKERHUB'] - publishGithubRelease: variables['PUBLISH_GITHUB_RELEASE'] diff --git a/.github/workflows/_publish_build.yml b/.github/workflows/_publish_build.yml new file mode 100644 index 0000000000000..857a2cf56b492 --- /dev/null +++ b/.github/workflows/_publish_build.yml @@ -0,0 +1,173 @@ +name: Build + +permissions: + contents: read + +on: + workflow_call: + secrets: + gpg-key: + required: true + gpg-key-password: + required: true + inputs: + request: + type: string + required: true + trusted: + type: boolean + required: true + +concurrency: + group: >- + ${{ github.actor != 'trigger-release-envoy[bot]' + && github.event.inputs.head_ref + || github.run_id + }}-${{ github.event.workflow.id }}-publish + cancel-in-progress: true + + +jobs: + binary: + permissions: + contents: read + packages: read + name: ${{ matrix.name || matrix.target }} + uses: ./.github/workflows/_run.yml + with: + bazel-extra: ${{ matrix.bazel-extra }} + target: ${{ matrix.target }} + cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }} + cache-build-image-key-suffix: ${{ matrix.arch == 'arm64' && format('-{0}', matrix.arch) || '' }} + concurrency-suffix: -${{ matrix.arch }} + rbe: ${{ matrix.rbe }} + request: ${{ inputs.request }} + runs-on: ${{ matrix.runs-on }} + timeout-minutes: 120 + trusted: ${{ inputs.trusted }} + upload-name: release.${{ matrix.arch }} + upload-path: envoy/${{ matrix.arch }}/bin/ + strategy: + fail-fast: false + matrix: + include: + - target: release.server_only + name: Release (x64) + arch: x64 + bazel-extra: >- + --config=rbe-envoy-engflow + rbe: true + runs-on: ubuntu-24.04 + - target: release.server_only + name: Release (arm64) + arch: arm64 + bazel-extra: >- + --config=cache-envoy-engflow + --config=bes-envoy-engflow + rbe: false + runs-on: envoy-arm64-medium + + distribution: + permissions: + contents: read + packages: read + secrets: + gpg-key: ${{ secrets.gpg-key }} + gpg-key-password: ${{ secrets.gpg-key-password }} + name: ${{ matrix.name || matrix.target }} + needs: + - binary + uses: ./.github/workflows/_run.yml + with: + bazel-extra: >- + --config=cache-envoy-engflow + --config=bes-envoy-engflow + downloads: | + release.${{ matrix.arch }}: release/${{ matrix.arch }}/bin/ + target: ${{ matrix.target }} + cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }} + cache-build-image-key-suffix: ${{ matrix.cache-build-image-key-suffix }} + concurrency-suffix: -${{ matrix.arch }} + import-gpg: true + rbe: false + request: ${{ inputs.request }} + runs-on: ${{ matrix.runs-on }} + trusted: ${{ inputs.trusted }} + upload-name: packages.${{ matrix.arch }} + upload-path: envoy/${{ matrix.arch }} + strategy: + fail-fast: false + matrix: + include: + - target: distribution + name: Package debs (x64) + arch: x64 + runs-on: ubuntu-24.04 + - target: distribution + name: Package debs (arm64) + arch: arm64 + cache-build-image-key-suffix: -arm64 + runs-on: envoy-arm64-small + + docker: + permissions: + contents: read + packages: read + name: ${{ matrix.name || matrix.target }} + needs: + - binary + uses: ./.github/workflows/_run.yml + with: + target: ${{ matrix.target }} + cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }} + downloads: | + release.arm64: envoy/arm64/bin/ + release.x64: envoy/x64/bin/ + request: ${{ inputs.request }} + runs-on: ${{ matrix.runs-on }} + source: | + export NO_BUILD_SETUP=1 + export ENVOY_DOCKER_IN_DOCKER=1 + trusted: ${{ inputs.trusted }} + upload-name: docker + upload-path: build_images + strategy: + fail-fast: false + matrix: + include: + - target: docker + name: Docker (Linux multiarch) + runs-on: ubuntu-24.04 + + sign: + permissions: + contents: read + packages: read + secrets: + gpg-key: ${{ secrets.gpg-key }} + gpg-key-password: ${{ secrets.gpg-key-password }} + name: ${{ matrix.name || matrix.target }} + needs: + - distribution + uses: ./.github/workflows/_run.yml + with: + target: release.signed + cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }} + downloads: | + packages.arm64: envoy/arm64/ + packages.x64: envoy/x64/ + release.arm64: envoy/arm64/bin/ + release.x64: envoy/x64/bin/ + import-gpg: true + request: ${{ inputs.request }} + runs-on: ubuntu-24.04 + source: | + export NO_BUILD_SETUP=1 + trusted: ${{ inputs.trusted }} + upload-name: release.signed + upload-path: envoy/release.signed.tar.zst + steps-pre: | + - run: | + mkdir distribution/custom + cp -a %{{ runner.temp }}/envoy/x64 %{{ runner.temp }}/envoy/arm64 distribution/custom + shell: bash diff --git a/.github/workflows/_publish_publish.yml b/.github/workflows/_publish_publish.yml index be26f8bdff4c8..36bb5f0fd4552 100644 --- a/.github/workflows/_publish_publish.yml +++ b/.github/workflows/_publish_publish.yml @@ -3,12 +3,6 @@ name: Publish permissions: contents: read -# The matrices in this config can be combined once the calling workflow has shifted -# to a `pull_request`/`commit` pattern (ie not `workflow_dispatch`) -# -# For now pre/post submit is split between `publish_ci`/`publish`, the latter running -# only for "trusted" runs and having access to secrets/permissions - on: workflow_call: secrets: @@ -47,6 +41,8 @@ jobs: target: ${{ matrix.target }} rbe: false cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }} + downloads: | + release.signed: release.signed source: ${{ matrix.source }} request: ${{ inputs.request }} steps-pre: ${{ matrix.steps-pre }} @@ -61,29 +57,6 @@ jobs: export ENVOY_COMMIT=${{ fromJSON(inputs.request).request.sha }} export ENVOY_REPO=${{ github.repository }} export ENVOY_PUBLISH_DRY_RUN=${{ (fromJSON(inputs.request).request.version.dev || ! inputs.trusted) && 1 || '' }} - steps-pre: | - - id: url - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.35 - with: - options: -Rr - input: >- - ${{ inputs.trusted - && fromJSON(inputs.request).request.sha - || fromJSON(inputs.request).request.ref }} - filter: | - .[:7] as $sha - | "release/release.signed.tar.zst" as $path - | if ${{ inputs.trusted }} then - "envoy-postsubmit" - else - "envoy-pr" - end - | . as $bucket - | "https://storage.googleapis.com/\($bucket)/\($sha)/\($path)" - - uses: envoyproxy/toolshed/gh-actions/fetch@actions-v0.2.35 - with: - url: %{{ steps.url.outputs.value }} - path: %{{ runner.temp }}/release.signed publish_docs: # For normal commits to Envoy main this will trigger an update in the website repo, diff --git a/.github/workflows/_publish_verify.yml b/.github/workflows/_publish_verify.yml index 075e4aad04407..09344b1da4976 100644 --- a/.github/workflows/_publish_verify.yml +++ b/.github/workflows/_publish_verify.yml @@ -23,7 +23,7 @@ concurrency: jobs: - verify-examples: + examples: permissions: contents: read packages: read @@ -35,9 +35,10 @@ jobs: cache-build-image-key-suffix: ${{ matrix.arch == 'arm64' && format('-{0}', matrix.arch) || '' }} container-command: ${{ matrix.container-command }} concurrency-suffix: -${{ matrix.arch || 'x64' }} + downloads: ${{ matrix.downloads }} rbe: ${{ matrix.rbe }} request: ${{ inputs.request }} - runs-on: ${{ matrix.runs-on || 'ubuntu-24.04' }} + runs-on: ubuntu-24.04 steps-pre: ${{ matrix.steps-pre }} source: ${{ matrix.source }} target: ${{ matrix.target }} @@ -48,6 +49,8 @@ jobs: include: - name: examples target: verify_examples + downloads: | + docker: build_images rbe: false source: | export NO_BUILD_SETUP=1 @@ -58,39 +61,24 @@ jobs: sudo apt-get -qq update -y sudo apt-get -qq install -y --no-install-recommends expect gettext yq whois shell: bash - - id: url - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.35 - with: - options: -Rr - input: >- - ${{ inputs.trusted - && fromJSON(inputs.request).request.sha - || fromJSON(inputs.request).request.ref }} - filter: | - .[:7] as $sha - | if ${{ inputs.trusted }} then - "envoy-postsubmit" - else - "envoy-pr" - end - | . as $bucket - | "https://storage.googleapis.com/\($bucket)/\($sha)" - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.35 - with: - url: %{{ steps.url.outputs.value }}/docker/envoy.tar - variant: dev - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.35 - with: - url: %{{ steps.url.outputs.value }}/docker/envoy-contrib.tar - variant: contrib-dev - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.35 - with: - url: %{{ steps.url.outputs.value }}/docker/envoy-google-vrp.tar - variant: google-vrp-dev + - run: | + IMAGES=( + envoy:dev + envoy-contrib:contrib-dev + envoy-google-vrp:google-vrp-dev) + for image in "${IMAGES[@]}"; do + src_name="$(echo ${image} | cut -d: -f1)" + dest_name="$(echo ${image} | cut -d: -f2)" + src="oci-archive:%{{ runner.temp }}/build_images/${src_name}.tar" + dest="docker-daemon:envoyproxy/envoy:${dest_name}" + echo "Copy image: ${src} ${dest}" + skopeo copy -q "${src}" "${dest}" + done + shell: bash - run: docker images | grep envoy shell: bash - verify-distro: + distro: permissions: contents: read packages: read @@ -102,6 +90,8 @@ jobs: cache-build-image-key-suffix: ${{ matrix.arch == 'arm64' && format('-{0}', matrix.arch) || '' }} container-command: ./ci/run_envoy_docker.sh concurrency-suffix: -${{ matrix.arch || 'x64' }} + downloads: | + release.signed: release.signed rbe: ${{ matrix.rbe && matrix.rbe || false }} request: ${{ inputs.request }} runs-on: ${{ matrix.runs-on || 'ubuntu-24.04' }} @@ -111,34 +101,13 @@ jobs: target: ${{ matrix.target }} trusted: ${{ inputs.trusted }} steps-pre: | - - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.30 - id: url - with: - options: -Rr - input: >- - ${{ inputs.trusted - && fromJSON(inputs.request).request.sha - || fromJSON(inputs.request).request.ref }} - filter: | - .[:7] as $sha - | if ${{ inputs.trusted }} then - "envoy-postsubmit" - else - "envoy-pr" - end - | . as $bucket - | "https://storage.googleapis.com/\($bucket)/\($sha)/release/release.signed.tar.zst" - - uses: envoyproxy/toolshed/gh-actions/fetch@actions-v0.2.30 - id: fetch - with: - url: %{{ steps.url.outputs.value }} - run: | echo ARCH=${{ matrix.arch || 'x64' }} >> $GITHUB_ENV - echo DEB_ARCH=${{ matrix.arch != 'arm64' && 'amd64' || 'arm64' }} >> $GITHUB_ENV + echo DEB_ARCH=${{ matrix.arch == 'arm64' && 'arm64' || 'amd64' }} >> $GITHUB_ENV shell: bash - run: | TEMP_DIR=$(mktemp -d) - zstd --stdout -d %{{ steps.fetch.outputs.path }} | tar --warning=no-timestamp -xf - -C "${TEMP_DIR}" + zstd --stdout -d %{{ runner.temp }}/release.signed/release.signed.tar.zst | tar --warning=no-timestamp -xf - -C "${TEMP_DIR}" mkdir ${TEMP_DIR}/debs tar xf ${TEMP_DIR}/bin/debs.tar.gz -C ${TEMP_DIR}/debs mkdir -p ${TEMP_DIR}/distribution/deb diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml index e65e87a4e2cbe..91667dc2f93d0 100644 --- a/.github/workflows/_run.yml +++ b/.github/workflows/_run.yml @@ -8,6 +8,8 @@ on: secrets: app-id: app-key: + gpg-key: + gpg-key-password: rbe-key: ssh-key: ssh-key-extra: @@ -59,6 +61,9 @@ on: Error: fail-match: type: string + import-gpg: + type: boolean + default: false notice-match: type: string default: | @@ -233,6 +238,16 @@ jobs: config: ${{ inputs.checkout-extra }} ssh-key: ${{ inputs.trusted && inputs.ssh-key-extra || '' }} + - uses: envoyproxy/toolshed/gh-actions/github/run@actions-v0.2.35 + - name: Import GPG key + uses: envoyproxy/toolshed/gh-actions/gpg/import@actions-v0.2.35 + if: ${{ inputs.import-gpg }} + with: + key: ${{ secrets.gpg-key }} + passphrase: ${{ secrets.gpg-key-password }} + passphrase-path: "${{ runner.temp }}/gpg-passphrase" + configured-passphrase-path: /build/gpg-passphrase + - uses: envoyproxy/toolshed/gh-actions/github/run@actions-v0.2.35 name: Run CI ${{ inputs.command }} ${{ inputs.target }} with: @@ -272,4 +287,6 @@ jobs: ${{ inputs.bazel-extra }} ${{ inputs.rbe == true && format('--jobs={0}', inputs.bazel-rbe-jobs) || '' }} BAZEL_FAKE_SCM_REVISION: ${{ github.event_name == 'pull_request' && 'e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9' || '' }} + CI_SHA1: ${{ github.sha }} CI_TARGET_BRANCH: ${{ fromJSON(inputs.request).request.target-branch }} + MOUNT_GPG_HOME: ${{ inputs.import-gpg && 1 || '' }} diff --git a/.github/workflows/envoy-publish.yml b/.github/workflows/envoy-publish.yml index df33cd5221bae..d4703d50bc1d5 100644 --- a/.github/workflows/envoy-publish.yml +++ b/.github/workflows/envoy-publish.yml @@ -50,6 +50,25 @@ jobs: check-name: publish head-sha: ${{ inputs.sha }} + build: + permissions: + contents: read + packages: read + secrets: + gpg-key: ${{ fromJSON(needs.load.outputs.trusted) && secrets.ENVOY_GPG_MAINTAINER_KEY || secrets.ENVOY_GPG_SNAKEOIL_KEY }} + gpg-key-password: >- + ${{ fromJSON(needs.load.outputs.trusted) + && secrets.ENVOY_GPG_MAINTAINER_KEY_PASSWORD + || secrets.ENVOY_GPG_SNAKEOIL_KEY_PASSWORD }} + if: ${{ fromJSON(needs.load.outputs.request).run.publish || fromJSON(needs.load.outputs.request).run.verify }} + needs: + - load + uses: ./.github/workflows/_publish_build.yml + name: Build + with: + request: ${{ needs.load.outputs.request }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + publish: secrets: ENVOY_CI_SYNC_APP_ID: ${{ fromJSON(needs.load.outputs.trusted) && secrets.ENVOY_CI_SYNC_APP_ID || '' }} @@ -62,6 +81,7 @@ jobs: if: ${{ fromJSON(needs.load.outputs.request).run.publish }} needs: - load + - build uses: ./.github/workflows/_publish_publish.yml name: Publish with: @@ -75,6 +95,7 @@ jobs: if: ${{ fromJSON(needs.load.outputs.request).run.verify }} needs: - load + - build uses: ./.github/workflows/_publish_verify.yml name: Verify with: @@ -95,6 +116,7 @@ jobs: || fromJSON(needs.load.outputs.request).run.verify) }} needs: - load + - build - publish - verify uses: ./.github/workflows/_finish.yml diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 038a7c7e4c06b..ab0c202d6b9d2 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -580,6 +580,7 @@ case $CI_TARGET in else ENVOY_RELEASE_TARBALL="/build/release/arm64/bin/release.tar.zst" fi + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ //tools/zstd \ -- --stdout \ @@ -598,8 +599,10 @@ case $CI_TARGET in --//distribution:envoy-contrib-binary=//distribution:custom/envoy-contrib \ //distribution:packages.tar.gz if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then + echo "Packages created: ${ENVOY_BUILD_DIR}/packages.x64.tar.gz" cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.x64.tar.gz" else + echo "Packages created: ${ENVOY_BUILD_DIR}/packages.x64.tar.gz" cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.arm64.tar.gz" fi ;; @@ -888,7 +891,9 @@ case $CI_TARGET in # Build echo "Building with:" + echo " build options: ${BAZEL_BUILD_OPTIONS[*]}" echo " release options: ${BAZEL_RELEASE_OPTIONS[*]}" + echo " binary dir: ${ENVOY_BINARY_DIR}" # Build release binaries bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ @@ -928,12 +933,6 @@ case $CI_TARGET in setup_clang_toolchain bazel build "${BAZEL_BUILD_OPTIONS[@]}" //distribution:signed cp -a bazel-bin/distribution/release.signed.tar.zst "${BUILD_DIR}/envoy/" - bazel run //tools/gcs:upload \ - "${GCS_ARTIFACT_BUCKET}" \ - "${GCP_SERVICE_ACCOUNT_KEY_PATH}" \ - "${BUILD_DIR}/envoy" \ - "release" \ - "${GCS_REDIRECT_PATH}" ;; sizeopt) diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 809d141fb9dab..4517d94bb3faa 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -92,6 +92,11 @@ VOLUMES=( -v "${ENVOY_DOCKER_BUILD_DIR}":"${BUILD_DIR_MOUNT_DEST}" -v "${SOURCE_DIR}":"${SOURCE_DIR_MOUNT_DEST}") +if [[ -n "$MOUNT_GPG_HOME" ]]; then + VOLUMES+=( + -v "${HOME}/.gnupg:${BUILD_DIR_MOUNT_DEST}/.gnupg") +fi + if ! is_windows; then export BUILD_DIR="${BUILD_DIR_MOUNT_DEST}" fi