diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 280ed141576cd..e0b1bc7859991 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -1,6 +1,15 @@ name: Eval -on: pull_request_target +on: + pull_request_target: + push: + # Keep this synced with ci/request-reviews/dev-branches.txt + branches: + - master + - staging + - release-* + - staging-* + - haskell-updates permissions: contents: read @@ -11,6 +20,7 @@ jobs: runs-on: ubuntu-latest outputs: mergedSha: ${{ steps.merged.outputs.mergedSha }} + baseSha: ${{ steps.baseSha.outputs.baseSha }} systems: ${{ steps.systems.outputs.systems }} steps: # Important: Because of `pull_request_target`, this doesn't check out the PR, @@ -24,14 +34,22 @@ jobs: id: merged env: GH_TOKEN: ${{ github.token }} + GH_EVENT: ${{ github.event_name }} run: | - if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then - echo "Checking the merge commit $mergedSha" - echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT" - else - # Skipping so that no notifications are sent - echo "Skipping the rest..." - fi + case "$GH_EVENT" in + push) + echo "mergedSha=${{ github.sha }}" >> "$GITHUB_OUTPUT" + ;; + pull_request_target) + if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then + echo "Checking the merge commit $mergedSha" + echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT" + else + # Skipping so that no notifications are sent + echo "Skipping the rest..." + fi + ;; + esac rm -rf base - name: Check out the PR at the test merge commit uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -41,6 +59,13 @@ jobs: ref: ${{ steps.merged.outputs.mergedSha }} path: nixpkgs + - name: Determine base commit + if: github.event_name == 'pull_request_target' && steps.merged.outputs.mergedSha + id: base + run: | + baseSha=$(git -C nixpkgs rev-parse HEAD^1) + echo "baseSha=$baseSha" >> "$GITHUB_OUTPUT" + - name: Install Nix uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 if: steps.merged.outputs.mergedSha @@ -124,18 +149,71 @@ jobs: - name: Combine all output paths and eval stats run: | nix-build nixpkgs/ci -A eval.combine \ - --arg resultsDir ./intermediate + --arg resultsDir ./intermediate \ + -o prResult - name: Upload the combined results uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: result - path: result/* + path: prResult/* + + - name: Get base run id + if: needs.attrs.outputs.baseSha + id: baseRunId + run: | + set -e + + # TODO: Wait until it's done + # Get the latest eval.yml workflow run for the PR's base commit + if ! run=$(gh api --method GET /repos/"$REPOSITORY"/actions/workflows/eval.yml/runs \ + -f head_sha="$BASE_SHA" \ + --jq '.workflow_runs | sort_by(.run_started_at) | .[-1]') + || [[ -z "$run" ]]; then + echo "Could not find an eval.yml workflow run for $BASE_SHA, cannot make comparison" + exit 0 + fi + echo "Comparing against $(jq .html_url <<< "$run")" + runId=$(jq .id <<< "$run") + conclusion=$(jq .conclusion <<< "$run") + + while [[ "$conclusion" == null ]]; do + echo "Workflow not done, waiting 10 seconds before checking again" + sleep 10 + conclusion=$(gh api /repos/"$REPOSITORY"/actions/runs/"$runId" --jq '.conclusion') + done + + if [[ "$conclusion" != "success" ]]; then + echo "Workflow was not successful, cannot make comparison" + exit 0 + fi + + echo "baseRunId=$runId" >> "$GITHUB_OUTPUT" + env: + REPOSITORY: ${{ github.repository }} + BASE_SHA: ${{ needs.attrs.outputs.baseSha }} + - uses: actions/download-artifact@v4 + if: steps.baseRunId.outputs.baseRunId + with: + name: result + path: baseResult + github-token: ${{ github.token }} + run-id: ${{ steps.baseRunId.outputs.baseRunId }} + + - name: Compare against the base branch + if: steps.baseRunId.outputs.baseRunId + run: | + nix-build nixpkgs/ci -A eval.compare \ + --arg beforeResultDir ./baseResult \ + --arg afterResultDir ./prResult \ + -o comparison - # TODO: Run this workflow also on `push` (on at least the main development branches) - # Then add an extra step here that waits for the base branch (not the merge base, because that could be very different) - # to have completed the eval, then use - # gh api --method GET /repos/NixOS/nixpkgs/actions/workflows/eval.yml/runs -f head_sha= - # and follow it to the artifact results, where you can then download the outpaths.json from the base branch - # That can then be used to compare the number of changed paths, get evaluation stats and ping appropriate reviewers + # TODO: Request reviews from maintainers for packages whose files are modified in the PR + + - name: Upload the combined results + if: steps.baseRunId.outputs.baseRunId + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: comparison + path: comparison/* diff --git a/ci/eval/default.nix b/ci/eval/default.nix index 3f31faef73ae3..f935056192712 100644 --- a/ci/eval/default.nix +++ b/ci/eval/default.nix @@ -238,6 +238,22 @@ let jq -s from_entries > $out/stats.json ''; + compare = + { beforeResultDir, afterResultDir }: + runCommand "compare" { + nativeBuildInputs = [ + jq + ]; + } '' + mkdir $out + jq -n -f ${./filter.jq} \ + --slurpfile before ${beforeResultDir}/outpaths.json \ + --slurpfile after ${afterResultDir}/outpaths.json \ + > $out/changed-paths.json + + # TODO: Compare eval stats + ''; + full = { # Whether to evaluate just a single system, by default all are evaluated @@ -262,12 +278,14 @@ let resultsDir = results; }; + in { inherit attrpathsSuperset singleSystem combine + compare # The above three are used by separate VMs in a GitHub workflow, # while the below is intended for testing on a single local machine full diff --git a/ci/request-reviews/dev-branches.txt b/ci/request-reviews/dev-branches.txt index 2282529881bab..b34092546f184 100644 --- a/ci/request-reviews/dev-branches.txt +++ b/ci/request-reviews/dev-branches.txt @@ -1,5 +1,6 @@ # Trusted development branches: # These generally require PRs to update and are built by Hydra. +# Keep this synced with the branches in .github/workflows/eval.yml master staging release-*