+13 - refactor, save sha only on pretest branch of fork #160
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Upstream pretest | |
on: | |
push: | |
schedule: | |
- cron: '0 0 * * *' | |
workflow_dispatch: | |
permissions: | |
contents: write | |
concurrency: | |
group: ${{ github.workflow }} | |
cancel-in-progress: false | |
# There are 2 steps of adding new upstream commits: | |
# 1. Add new commits from the PR head ref (and, if needed, main/custom branches): to "pretest/$pr_num" in fork | |
# 2. Add commit with test changes: to "subm-pretest/$org_repo/$pr_num" in POCGL repo | |
# To only update when source has been updated: | |
# - "pretest" references sha of latest PR commit as: meta.pr_head_sha | |
# - "pretest" references sha of latest "subm-pretest" commit as: meta.core_pretest_sha | |
# "pretest" is not updated when only main or custom branch of subm updates | |
# - Otherwise all pretest branches would update at the same time, even on unrelated changes | |
# "pretest" needs to be first checked by a separate "git ls-remote" before "git fetch" | |
# - Otherwise git fetch would fail fetching the missing branches | |
# "pretest" and "subm-pretest" can be update manually by push, in case anything new needs to be implemented for the new XML spec | |
# - In that case, there would be no commit sha reference, so it will be re-tested on the next run of this workflow | |
jobs: | |
enmr-PRs: | |
runs-on: windows-latest | |
outputs: | |
exec_list: ${{ steps.make_exec_list.outputs.exec_list }} | |
remove_list: ${{ steps.make_exec_list.outputs.remove_list }} | |
steps: | |
- name: git config | |
run: | | |
git config --global gc.auto 0 | |
git config --global core.autocrlf false | |
git config --global user.name "sun pack bot" | |
git config --global user.email "[email protected]" | |
git config --global --add url.https://github.com/.insteadOf "[email protected]:" | |
- name: checkout | |
uses: actions/checkout@main | |
with: | |
sparse-checkout-cone-mode: false | |
sparse-checkout: | | |
.gitmodules | |
- name: (!) make exec list | |
id: make_exec_list | |
run: | | |
function Get-GitRemoteBranches() { | |
param ($url, $pattern) | |
$reg_pattern = $pattern -replace '\*','(.*)' | |
$res = @() | |
foreach ($l in git ls-remote $url $pattern) { | |
if ($l -notmatch "(^[0-9a-f]{40})\s+($reg_pattern)$") { | |
throw "Unexpected git ls-remote output: $l" | |
} | |
$sha = $matches[1] | |
$branch = $matches[2] | |
if ($branch -notmatch "^$reg_pattern$") { | |
throw "What?" | |
} | |
$res += [PSCustomObject]@{ | |
'sha' = $sha; | |
'branch' = $branch; | |
'matches' = $matches; | |
} | |
} | |
if (-not $?) { throw "git ls-remote failed" } | |
return $res | |
} | |
Write-Host "==============================" | |
Write-Host "Bulding open PR list" | |
$submodules = @() | |
$submodule_by_repo = @{} | |
foreach ($config_key in git config --file .gitmodules --name-only --get-regexp '^submodule\..+\.url$') { | |
$url = git config --file .gitmodules --get $config_key | |
Write-Host "URL: $url" | |
if ($url -notmatch '^git@github\.com:([\w\-]+)/([\w\-]+)\.git$') { | |
throw "Unexpected url format: $url" | |
} | |
$owner = $matches[1] | |
$name = $matches[2] | |
$fork_url = "[email protected]:SunSerega/${name}.git" | |
$mergeable_prs = @() | |
foreach ($l in Get-GitRemoteBranches $url 'refs/pull/*/merge') { | |
$pr_num = $l.matches[1] | |
$mergeable_prs += $pr_num | |
} | |
$head_sha_by_pr = @{} | |
foreach ($l in Get-GitRemoteBranches $url 'refs/pull/*/head') { | |
$sha = $l.sha | |
$pr_num = $l.matches[1] | |
if ($pr_num -notin $mergeable_prs) { continue } | |
Write-Host "- Found open PR $pr_num with latest sha $sha" | |
$head_sha_by_pr.Add($pr_num, $sha) | |
} | |
$pretest_prs = @{} | |
foreach ($l in Get-GitRemoteBranches $fork_url 'refs/heads/pretest/*') { | |
$sha = $l.sha | |
$pr_num = $l.matches[1] | |
Write-Host "- Found pretest branch for PR $pr_num with latest sha $sha" | |
$pretest_prs += $pr_num | |
} | |
$subm = [PSCustomObject]@{ | |
'url' = $url; | |
'fork_url' = $fork_url; | |
'owner' = $owner; | |
'name' = $name; | |
'repo' = "$owner/$name" | |
'head_sha_by_pr' = $head_sha_by_pr; | |
'pretest_prs' = $pretest_prs; | |
} | |
$submodules += $subm | |
$submodule_by_repo.Add($subm.repo, $subm) | |
} | |
Write-Host "==============================" | |
Write-Host "Fetching extra data" | |
$core_pretest_pr_nums = @{} | |
$subm_fetch_specs = @{} | |
foreach ($l in Get-GitRemoteBranches origin 'refs/heads/subm-pretest/*/*/*') { | |
$sha = $l.sha | |
$subm_owner = $l.matches[1] | |
$subm_name = $l.matches[2] | |
$pr_num = $l.matches[3] | |
$repo = "$subm_owner/$subm_name" | |
Write-Host "- Found core pretest branch for repo $repo PR $pr_num" | |
$core_pretest_pr_nums[$repo] += @{ "$pr_num" = $sha } | |
if ($pr_num -in $submodule_by_repo[$repo].pretest_prs) { | |
$fork_pretest_branch = "pretest/$pr_num" | |
$subm_fetch_specs[$repo] += @("refs/heads/${fork_pretest_branch}:${fork_pretest_branch}") | |
} else { | |
Write-Host "--- MISSING FORK PRETEST" | |
} | |
} | |
# To check which pretest branches are up to date and should not be tested | |
# - Only need commit messages, not files | |
foreach ($repo in $submodule_by_repo.Keys) { | |
if (-not $subm_fetch_specs[$repo]) { continue } | |
Write-Host "- Fetching branches for $repo" | |
# Write-Host git fetch --depth=1 --no-tags --filter=tree:0 $submodule_by_repo[$repo].fork_url $subm_fetch_specs[$repo] | |
git fetch --depth=1 --no-tags --filter=tree:0 $submodule_by_repo[$repo].fork_url $subm_fetch_specs[$repo] | |
if (-not $?) { throw "git fetch failed" } | |
} | |
Write-Host "==============================" | |
Write-Host "Building exec_list" | |
$exec_list = @() | |
foreach ($subm in $submodules) { | |
Write-Host "subm: $($subm.name)" | |
foreach ($pr_num in $subm.head_sha_by_pr.Keys) { | |
Write-Host "- PR $pr_num" | |
function Is-SkipNeeded() { | |
if ($pr_num -notin $subm.pretest_prs) { return $false } | |
if ($pr_num -notin $core_pretest_pr_nums[$subm.repo].Keys) { return $false } | |
$extra_debug = $true | |
$pr_head_sha_found = $false | |
$core_pretest_sha_found = $false | |
foreach ($l in git log -1 --pretty=format:"%B" "pretest/$pr_num") { | |
if ($extra_debug) { | |
Write-Host "--- $l" | |
} | |
if ($l -notmatch '^meta\.(\w+)=([0-9a-f]{40})$') { continue } | |
$key = $matches[1] | |
$sha = $matches[2] | |
if ($extra_debug) { | |
Write-Host "----- FOUND [$key]: $sha" | |
} | |
if ($key -eq 'pr_head_sha') { | |
if ($sha -eq $subm.head_sha_by_pr[$pr_num]) { | |
$pr_head_sha_found = $true | |
} elseif ($extra_debug) { | |
Write-Host "----- SHA unexpected" | |
} | |
} elseif ($key -eq 'core_pretest_sha') { | |
if ($sha -eq $core_pretest_pr_nums[$subm.repo][$pr_num]) { | |
$core_pretest_sha_found = $true | |
} elseif ($extra_debug) { | |
Write-Host "----- SHA unexpected" | |
} | |
} elseif ($extra_debug) { | |
Write-Host "----- Key unexpected" | |
} | |
} | |
return $pr_head_sha_found -and $core_pretest_sha_found | |
} | |
if (Is-SkipNeeded) { continue } | |
Write-Host "--- TEST NOW" | |
$exec_list += [PSCustomObject]@{ | |
'subm_name' = $subm.name; | |
'org_repo' = $subm.repo; | |
'fork_repo' = "SunSerega/$($subm.name)"; | |
'pr_num' = $pr_num; | |
} | |
} | |
} | |
$json = ConvertTo-Json -Compress $exec_list | |
Write-Host $json | |
echo "exec_list=$json" >> $env:GITHUB_OUTPUT | |
Write-Host "==============================" | |
Write-Host "Building remove_list" | |
$remove_list = @() | |
foreach ($subm in $submodules) { | |
Write-Host "subm: $($subm.name)" | |
$subm_rem_pretest_pr_nums = @() | |
foreach ($pr_num in $subm.pretest_prs) { | |
Write-Host "- Fork pretest for PR $pr_num" | |
if ($pr_num -in $subm.pr_head_sha.Keys) { continue } | |
$subm_rem_pretest_pr_nums += $pr_num | |
Write-Host "--- REMOVE" | |
} | |
$core_rem_pretest_pr_nums = @() | |
foreach ($pr_num in $core_pretest_pr_nums[$subm.repo].Keys) { | |
Write-Host "- Core pretest for PR $pr_num" | |
if ($pr_num -in $subm.pr_head_sha.Keys) { continue } | |
$core_rem_pretest_pr_nums += $pr_num | |
} | |
if ($subm_rem_pretest_pr_nums -or $core_rem_pretest_pr_nums) { | |
$remove_list += [PSCustomObject]@{ | |
'org_repo' = $subm.repo; | |
'fork_repo' = "SunSerega/$($subm.name)"; | |
'subm_pretest_pr_nums' = $subm_rem_pretest_pr_nums -join ','; | |
'core_pretest_pr_nums' = $core_rem_pretest_pr_nums -join ','; | |
} | |
} | |
} | |
$json = ConvertTo-Json -Compress $remove_list | |
Write-Host $json | |
echo "remove_list=$json" >> $env:GITHUB_OUTPUT | |
pretest-each: | |
runs-on: windows-latest | |
needs: enmr-PRs | |
if: ${{ needs.enmr-PRs.outputs.exec_list != '[]' }} | |
strategy: | |
fail-fast: false | |
matrix: | |
exec-data: ${{ fromJson(needs.enmr-PRs.outputs.exec_list) }} | |
steps: | |
- name: git config | |
run: | | |
git config --global gc.auto 0 | |
git config --global core.autocrlf false | |
git config --global advice.detachedHead false | |
git config --global user.name "sun pack bot" | |
git config --global user.email "[email protected]" | |
git config --global --add url.https://github.com/.insteadOf "[email protected]:" | |
- name: checkout main repo | |
uses: actions/checkout@main | |
with: | |
ref: ${{ github.event.repository.default_branch }} | |
path: './core' | |
fetch-depth: 0 | |
- name: checkout subm fork | |
uses: actions/checkout@main | |
with: | |
ref: 'custom' | |
path: './fork' | |
fetch-depth: 0 | |
repository: ${{ matrix.exec-data.fork_repo }} | |
token: ${{ secrets.POCGL_pretest_upstream_PAT }} | |
- name: (!) pretest | |
run: | | |
$subm_name = '${{ matrix.exec-data.subm_name }}' | |
$org_repo = '${{ matrix.exec-data.org_repo }}' | |
$fork_repo = '${{ matrix.exec-data.fork_repo }}' | |
$pr_num = "${{ matrix.exec-data.pr_num }}" | |
$remote_official = '0_official' | |
$b_fork_upstream_main = "$remote_official/main" | |
$b_fork_main = 'custom' | |
$b_fork_pretest = "pretest/$pr_num" | |
$b_fork_merged_pr_head = 'merged_pr_head' # $pr_num/head + main | |
$b_fork_virgin_merge_test = 'virgin_merge_test' # $b_merged_pr_head + custom | |
$b_core_main = '${{ github.event.repository.default_branch }}' | |
$b_core_subm_pretest = "subm-pretest/$org_repo/$pr_num" | |
Write-Host "Updating subm fork..." | |
Push-Location './fork' | |
git remote add $remote_official "[email protected]:${org_repo}.git" | |
git config --add "remote.$remote_official.fetch" "+refs/pull/*:refs/remotes/$remote_official/pull/*" | |
git fetch --all 2>&1 | Out-Null | |
Write-Host "- Creating $b_fork_merged_pr_head" | |
git checkout "remotes/$remote_official/pull/$pr_num/head" | |
if (-not $?) { throw "git checkout failed" } | |
git checkout -b $b_fork_merged_pr_head | |
if (-not $?) { throw "git checkout -b failed" } | |
Write-Host "--- Merging with ${b_fork_upstream_main}:" | |
git merge $b_fork_upstream_main | |
if (-not $?) { | |
Write-Host "git merge failed" | |
Write-Host "This PR is outdated and cannot be properly tested" | |
exit 0 | |
} | |
Write-Host "- Creating $b_fork_virgin_merge_test" | |
git checkout -b $b_fork_virgin_merge_test | |
if (-not $?) { throw "git checkout -b failed" } | |
$virgin_merge_test_exists = $true | |
Write-Host "--- Merging with ${b_fork_main}:" | |
git merge $b_fork_main | |
if (-not $?) { | |
Write-Host "git merge $b_fork_main failed" | |
Write-Host "Cannot create $b_fork_virgin_merge_test" | |
$virgin_merge_test_exists = $false | |
} | |
if (&{ git show-ref --verify -q "refs/remotes/origin/$b_fork_pretest"; $? }) { | |
Write-Host "- Branch for PR $pr_num exists" | |
git checkout $b_fork_pretest | |
Write-Host "--- Merging with ${b_fork_merged_pr_head}:" | |
git merge $b_fork_merged_pr_head | |
if (-not $?) { throw "git merge failed" } | |
Write-Host "--- Merging with ${b_fork_main}:" | |
git merge $b_fork_main | |
if (-not $?) { throw "git merge failed" } | |
$need_push = $true | |
if ($virgin_merge_test_exists) { | |
Write-Host "--- Diff with ${b_fork_virgin_merge_test}:" | |
git diff HEAD $b_fork_virgin_merge_test --exit-code | |
if (-not $?) { | |
Write-Host "----- Diff found, building on top" | |
} else { | |
Write-Host "----- Diff not found, resetting branch" | |
git reset --hard $b_fork_virgin_merge_test | |
if (-not $?) { throw "git reset failed" } | |
git push -f | |
if (-not $?) { throw "git push -f failed" } | |
$need_push = $false | |
} | |
} | |
if ($need_push) { | |
git push | |
if (-not $?) { throw "git push failed" } | |
} | |
} else { | |
Write-Host "- Branch for PR $pr_num doesn't exist" | |
if (-not $virgin_merge_test_exists) { | |
throw "Cannot do anything without $b_fork_virgin_merge_test" | |
} | |
Write-Host "--- Creating:" | |
git checkout $b_fork_virgin_merge_test | |
if (-not $?) { throw "git checkout failed" } | |
git checkout -b $b_fork_pretest | |
if (-not $?) { throw "git checkout -b failed" } | |
git push --set-upstream origin $b_fork_pretest | |
if (-not $?) { throw "git push failed" } | |
} | |
Pop-Location | |
Write-Host "Updating main repo..." | |
Push-Location './core' | |
git fetch --all 2>&1 | Out-Null | |
if (&{ git show-ref --verify -q "refs/remotes/origin/$b_core_subm_pretest"; $? }) { | |
Write-Host "- Branch for PR $pr_num exists" | |
git checkout $b_core_subm_pretest | |
if (-not $?) { throw "git checkout failed" } | |
Write-Host "--- Deleting the latest [trivial] commits:" | |
while ($true) { | |
($commit_name = (git log -1 --pretty=format:"%B" HEAD) -join "`n") | |
if (-not $?) { throw "git log failed" } | |
if (-not $commit_name.Contains("[trivial]")) { break } | |
Write-Host "=====" | |
git reset --hard HEAD~1 | |
if (-not $?) { throw "git reset failed" } | |
} | |
# expecting fast-forward when possible | |
Write-Host "--- Merging with ${b_core_main}:" | |
git merge $b_core_main -m "[trivial] Merge $b_core_main into subm-pretest/..." | |
if (-not $?) { throw "git merge failed" } | |
git push | |
if (-not $?) { throw "git push failed" } | |
} else { | |
Write-Host "- Branch for PR $pr_num doesn't exist" | |
Write-Host "--- Creating:" | |
git checkout -b $b_core_subm_pretest | |
if (-not $?) { throw "git checkout -b failed" } | |
git push --set-upstream origin $b_core_subm_pretest | |
if (-not $?) { throw "git push failed" } | |
} | |
$core_pretest_sha = git rev-parse HEAD | |
Pop-Location | |
Write-Host "Adding meta to fork repo..." | |
Push-Location './fork' | |
git commit --allow-empty -m @" | |
Add pretest meta | |
meta.pr_head_sha=$pr_head_sha | |
meta.core_pretest_sha=$core_pretest_sha | |
"@ | |
if (-not $?) { throw "git commit failed" } | |
git push | |
if (-not $?) { throw "git push failed" } | |
Pop-Location | |
& { | |
Write-Host "Installing OpenCL driver for Intel CPU" | |
# https://www.intel.com/content/www/us/en/developer/articles/technical/intel-cpu-runtime-for-opencl-applications-with-sycl-support.html | |
Invoke-WebRequest -Uri 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0e6849e6-2c56-480b-afcf-be8331d5c4f6-opencl/w_opencl_runtime_p_2024.1.0.968.exe' -OutFile 'D:\igfx.exe' | |
7z x "D:\igfx.exe" -o"D:\igfx" -y | |
D:\igfx\w_opencl_runtime_p_2024.1.0.968.msi /quiet | |
} | |
& { | |
Write-Host "Downloading and unpacking Pascal compiler" | |
Invoke-WebRequest -Uri 'https://github.com/SunSerega/pascalabcnet/releases/download/custom-build-tag/PABCNETC.zip' -OutFile 'D:\PABCNETC.zip' | |
Expand-Archive -Path 'D:\PABCNETC.zip' -DestinationPath 'D:\PABCNETC' -Force | |
} | |
Write-Host "Testing main repo..." | |
Push-Location './core' | |
& .\DeleteAllTemp.bat NoPause | Out-Null | |
Write-Host "- Compile: " -NoNewline | |
Start-Process -FilePath 'D:\PABCNETC\pabcnetcclear.exe' -ArgumentList '"PackAll.pas"' -Wait -NoNewWindow | |
& .\PackAll.exe "Stages= PullUpstream + Reference + Dummy + OpenCL+OpenCLABC + OpenGL+OpenGLABC + Compile + Test + Release" "PullUpstreamBranch=${subm_name}:${b_fork_pretest}" "PasCompPath=D:\PABCNETC\pabcnetcclear.exe" SkipFinishedPause | |
if (-not $?) { throw "PackAll failed" } | |
$need_commit = (git diff --name-only ":!LastPack.log" ":!Log/PullUpstream.log" ":!DataScraping/Reps") -or (git ls-files --others --exclude-standard) | |
if ($need_commit) { | |
Write-Host "- Found significant changes" | |
git commit -a -m '[trivial] test changes' | |
if (-not $?) { throw "git commit failed" } | |
git push | |
if (-not $?) { throw "git push failed" } | |
} | |
Pop-Location | |
remove-old-pretest: | |
runs-on: windows-latest | |
needs: enmr-PRs | |
if: ${{ needs.enmr-PRs.outputs.remove_list != '[]' }} | |
strategy: | |
fail-fast: false | |
matrix: | |
remove-data: ${{ fromJson(needs.enmr-PRs.outputs.remove_list) }} | |
steps: | |
- name: git config | |
run: | | |
git config --global gc.auto 0 | |
git config --global core.autocrlf false | |
git config --global user.name "sun pack bot" | |
git config --global user.email "[email protected]" | |
git config --global --add url.https://github.com/.insteadOf "[email protected]:" | |
- name: checkout main repo | |
uses: actions/checkout@main | |
with: | |
path: './core' | |
fetch-depth: 0 | |
sparse-checkout-cone-mode: false | |
sparse-checkout: '.gitmodules' | |
- name: checkout subm fork | |
uses: actions/checkout@main | |
with: | |
path: './fork' | |
sparse-checkout-cone-mode: false | |
sparse-checkout: '<NOTHING>' | |
repository: ${{ matrix.remove-data.fork_repo }} | |
token: ${{ secrets.POCGL_pretest_upstream_PAT }} | |
- name: (!) remove old pretest branches | |
run: | | |
function Split-OrEmpty { | |
param ( $s ) | |
if (-not $s) { return @() } | |
return $s -split ',' | |
} | |
$org_repo = '${{ matrix.remove-data.org_repo }}' | |
$fork_repo = '${{ matrix.remove-data.fork_repo }}' | |
$subm_pretest_pr_nums = Split-OrEmpty '${{ matrix.remove-data.subm_pretest_pr_nums }}' | |
$core_pretest_pr_nums = Split-OrEmpty '${{ matrix.remove-data.core_pretest_pr_nums }}' | |
Push-Location './core' | |
$core_removed_pr_nums = @() | |
foreach ($pr_num in $core_pretest_pr_nums) { | |
Write-Host "Checking if subm-pretest branch for PR $pr_num is trivial" | |
$branch_name = "subm-pretest/$org_repo/$pr_num" | |
Write-Host $branch_name | |
$is_trivial = $true | |
foreach ($commit_name in git show --pretty=format:"%s" -s "HEAD..origin/$branch_name") { | |
Write-Host "- $commit_name" | |
if ($commit_name.StartsWith('[trivial]')) { continue } | |
Write-Host "--- NOT TRIVIAL" | |
$is_trivial = $false | |
# break | |
} | |
if (-not $?) { throw "git show failed" } | |
if ($is_trivial) { | |
Write-Host "Is trivial, removing..." | |
git push origin --delete $branch_name # BRANCH DELETE | |
if (-not $?) { throw "git push --delete failed" } | |
$core_removed_pr_nums += $pr_num | |
} | |
Write-Host '~~~~~' | |
} | |
Pop-Location | |
$core_pretest_pr_nums = $core_pretest_pr_nums | | |
Where-Object { $_ -notin $core_removed_pr_nums } | |
Push-Location './fork' | |
foreach ($pr_num in $subm_pretest_pr_nums) { | |
Write-Host "PR $pr_num was closed" | |
if ($core_pretest_pr_nums -contains $pr_num) { | |
Write-Host "But main repo is still referencing it" | |
} else { | |
Write-Host "Removing..." | |
git push origin --delete "pretest/$pr_num" # BRANCH DELETE | |
if (-not $?) { throw "git push --delete failed" } | |
} | |
Write-Host '~~~~~' | |
} | |
Pop-Location | |
if ($core_pretest_pr_nums) { | |
Write-Host "pretest branches for these PRs are dangling:" | |
foreach ($pr_num in $core_pretest_pr_nums) { | |
Write-Host "- $pr_num" | |
} | |
throw "Some pretest branches in main repo are dangling" | |
} | |