Auto merge of #132526 - bjorn3:sync_cg_clif-2024-11-02, r=bjorn3 #52167
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This file defines our primary CI workflow that runs on pull requests | |
| # and also on pushes to special branches (auto, try). | |
| # | |
| # The actual definition of the executed jobs is calculated by a Python | |
| # script located at src/ci/github-actions/calculate-job-matrix.py, which | |
| # uses job definition data from src/ci/github-actions/jobs.yml. | |
| # You should primarily modify the `jobs.yml` file if you want to modify | |
| # what jobs are executed in CI. | |
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - auto | |
| - try | |
| - try-perf | |
| - automation/bors/try | |
| pull_request: | |
| branches: | |
| - "**" | |
| permissions: | |
| contents: read | |
| packages: write | |
| defaults: | |
| run: | |
| # On Linux, macOS, and Windows, use the system-provided bash as the default | |
| # shell. (This should only make a difference on Windows, where the default | |
| # shell is PowerShell.) | |
| shell: bash | |
| concurrency: | |
| # For a given workflow, if we push to the same branch, cancel all previous builds on that branch. | |
| # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which | |
| # are all triggered on the same branch, but which should be able to run concurrently. | |
| group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" | |
| # This will be empty in PR jobs. | |
| TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} | |
| jobs: | |
| # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml. | |
| # It calculates which jobs should be executed, based on the data of the ${{ github }} context. | |
| # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml. | |
| calculate_matrix: | |
| name: Calculate job matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| jobs: ${{ steps.jobs.outputs.jobs }} | |
| run_type: ${{ steps.jobs.outputs.run_type }} | |
| steps: | |
| - name: Checkout the source code | |
| uses: actions/checkout@v4 | |
| - name: Calculate the CI job matrix | |
| env: | |
| COMMIT_MESSAGE: ${{ github.event.head_commit.message }} | |
| run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT | |
| id: jobs | |
| job: | |
| name: ${{ matrix.name }} | |
| needs: [ calculate_matrix ] | |
| runs-on: "${{ matrix.os }}" | |
| defaults: | |
| run: | |
| shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }} | |
| timeout-minutes: 240 | |
| env: | |
| CI_JOB_NAME: ${{ matrix.image }} | |
| CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse | |
| # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs. | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
| DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SCCACHE_BUCKET: rust-lang-ci-sccache2 | |
| CACHE_DOMAIN: ci-caches.rust-lang.org | |
| continue-on-error: ${{ matrix.continue_on_error || false }} | |
| strategy: | |
| matrix: | |
| # Check the `calculate_matrix` job to see how is the matrix defined. | |
| include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }} | |
| steps: | |
| - if: contains(matrix.os, 'windows') | |
| uses: msys2/[email protected] | |
| with: | |
| # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64. | |
| msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }} | |
| # don't try to download updates for already installed packages | |
| update: false | |
| # don't try to use the msys that comes built-in to the github runner, | |
| # so we can control what is installed (i.e. not python) | |
| release: true | |
| # Inherit the full path from the Windows environment, with MSYS2's */bin/ | |
| # dirs placed in front. This lets us run Windows-native Python etc. | |
| path-type: inherit | |
| install: > | |
| make | |
| - name: disable git crlf conversion | |
| run: git config --global core.autocrlf false | |
| - name: checkout the source code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| # Free up disk space on Linux by removing preinstalled components that | |
| # we do not need. We do this to enable some of the less resource | |
| # intensive jobs to run on free runners, which however also have | |
| # less disk space. | |
| - name: free up disk space | |
| uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be | |
| if: contains(matrix.os, 'ubuntu') | |
| with: | |
| # Removing packages with APT saves ~5 GiB, but takes several | |
| # minutes (and potentially removes important packages). | |
| large-packages: false | |
| # Rust Log Analyzer can't currently detect the PR number of a GitHub | |
| # Actions build on its own, so a hint in the log message is needed to | |
| # point it in the right direction. | |
| - name: configure the PR in which the error message will be posted | |
| run: echo "[CI_PR_NUMBER=$num]" | |
| env: | |
| num: ${{ github.event.number }} | |
| if: needs.calculate_matrix.outputs.run_type == 'pr' | |
| - name: add extra environment variables | |
| run: src/ci/scripts/setup-environment.sh | |
| env: | |
| # Since it's not possible to merge `${{ matrix.env }}` with the other | |
| # variables in `job.<name>.env`, the variables defined in the matrix | |
| # are passed to the `setup-environment.sh` script encoded in JSON, | |
| # which then uses log commands to actually set them. | |
| EXTRA_VARIABLES: ${{ toJson(matrix.env) }} | |
| - name: setup upstream remote | |
| run: src/ci/scripts/setup-upstream-remote.sh | |
| - name: ensure the channel matches the target branch | |
| run: src/ci/scripts/verify-channel.sh | |
| - name: collect CPU statistics | |
| run: src/ci/scripts/collect-cpu-stats.sh | |
| - name: show the current environment | |
| run: src/ci/scripts/dump-environment.sh | |
| - name: install awscli | |
| run: src/ci/scripts/install-awscli.sh | |
| - name: install sccache | |
| run: src/ci/scripts/install-sccache.sh | |
| - name: select Xcode | |
| run: src/ci/scripts/select-xcode.sh | |
| - name: install clang | |
| run: src/ci/scripts/install-clang.sh | |
| - name: install tidy | |
| run: src/ci/scripts/install-tidy.sh | |
| - name: install WIX | |
| run: src/ci/scripts/install-wix.sh | |
| - name: disable git crlf conversion | |
| run: src/ci/scripts/disable-git-crlf-conversion.sh | |
| - name: checkout submodules | |
| run: src/ci/scripts/checkout-submodules.sh | |
| - name: install MinGW | |
| run: src/ci/scripts/install-mingw.sh | |
| - name: install ninja | |
| run: src/ci/scripts/install-ninja.sh | |
| - name: enable ipv6 on Docker | |
| run: src/ci/scripts/enable-docker-ipv6.sh | |
| # Disable automatic line ending conversion (again). On Windows, when we're | |
| # installing dependencies, something switches the git configuration directory or | |
| # re-enables autocrlf. We've not tracked down the exact cause -- and there may | |
| # be multiple -- but this should ensure submodules are checked out with the | |
| # appropriate line endings. | |
| - name: disable git crlf conversion | |
| run: src/ci/scripts/disable-git-crlf-conversion.sh | |
| - name: ensure line endings are correct | |
| run: src/ci/scripts/verify-line-endings.sh | |
| - name: ensure backported commits are in upstream branches | |
| run: src/ci/scripts/verify-backported-commits.sh | |
| - name: ensure the stable version number is correct | |
| run: src/ci/scripts/verify-stable-version-number.sh | |
| - name: run the build | |
| # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs. | |
| run: src/ci/scripts/run-build-from-ci.sh 2>&1 | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }} | |
| - name: create github artifacts | |
| run: src/ci/scripts/create-doc-artifacts.sh | |
| - name: print disk usage | |
| run: | | |
| echo "disk usage:" | |
| df -h | |
| - name: upload artifacts to github | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| # name is set in previous step | |
| name: ${{ env.DOC_ARTIFACT_NAME }} | |
| path: obj/artifacts/doc | |
| if-no-files-found: ignore | |
| retention-days: 5 | |
| - name: upload artifacts to S3 | |
| run: src/ci/scripts/upload-artifacts.sh | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }} | |
| # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy | |
| # builders *should* have the AWS credentials available. Still, explicitly | |
| # adding the condition is helpful as this way CI will not silently skip | |
| # deploying artifacts from a dist builder if the variables are misconfigured, | |
| # erroring about invalid credentials instead. | |
| if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1' | |
| - name: upload job metrics to DataDog | |
| if: needs.calculate_matrix.outputs.run_type != 'pr' | |
| env: | |
| DATADOG_SITE: datadoghq.com | |
| DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} | |
| DD_GITHUB_JOB_NAME: ${{ matrix.name }} | |
| run: | | |
| npm install -g @datadog/datadog-ci@^2.x.x | |
| python3 src/ci/scripts/upload-build-metrics.py build/cpu-usage.csv | |
| # This job isused to tell bors the final status of the build, as there is no practical way to detect | |
| # when a workflow is successful listening to webhooks only in our current bors implementation (homu). | |
| outcome: | |
| name: bors build finished | |
| runs-on: ubuntu-latest | |
| needs: [ calculate_matrix, job ] | |
| # !cancelled() executes the job regardless of whether the previous jobs passed or failed | |
| if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }} | |
| steps: | |
| - name: checkout the source code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| # Calculate the exit status of the whole CI workflow. | |
| # If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully). | |
| # If a some dependent job has failed, this exits with 1. | |
| - name: calculate the correct exit status | |
| run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}' | |
| # Publish the toolstate if an auto build succeeds (just before push to master) | |
| - name: publish toolstate | |
| run: src/ci/publish_toolstate.sh | |
| shell: bash | |
| if: needs.calculate_matrix.outputs.run_type == 'auto' | |
| env: | |
| TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues | |
| TOOLSTATE_PUBLISH: 1 |