From e7a9a8073827a4edfad969a105ca76379826c08b Mon Sep 17 00:00:00 2001 From: Sun Serega Date: Sat, 31 Aug 2024 16:43:15 +0200 Subject: [PATCH] +13 - refactor, save sha only on pretest branch of fork --- .github/workflows/upstream pretest.yaml | 367 +++++++++++++++--------- 1 file changed, 224 insertions(+), 143 deletions(-) diff --git a/.github/workflows/upstream pretest.yaml b/.github/workflows/upstream pretest.yaml index d0edb8c1..83d69267 100644 --- a/.github/workflows/upstream pretest.yaml +++ b/.github/workflows/upstream pretest.yaml @@ -13,30 +13,19 @@ concurrency: group: ${{ github.workflow }} cancel-in-progress: false -# TODO delete trivial subm-pretest before testing -# - One last commit at a time -# TODO find a way to hide upstream-only changes -# - Store sha on the pretest side, not subm-pretest -# TODO don't update pretest, when only the /merge has changed -# - Merge the /merge commit, but test the /head commit -# - No, actually, better way would be to always first try pull /head and merge main into it, and then merge that into the current state of pretest branch -# - That way I can always first check for upstream conflicts - - - # There are 2 steps of adding new upstream commits: -# 1. Add new commits from the PR merge ref (and possibly custom branch): to "pretest/$pr_num" in fork -# 2. Add commit with submodule ref move + test changes, with this ref move: to "subm-pretest/$org_repo/$pr_num" in POCGL repo +# 1. Add new commits from the PR head ref (and, if needed, main/custom branches): to "pretest/$pr_num" in fork +# 2. Add commit with test changes: to "subm-pretest/$org_repo/$pr_num" in POCGL repo # To only update when source has been updated: -# - "pretest" references sha of latest merge commit for PR as: meta.pr_merge_sha -# - "subm-pretest" references sha of latest pretest commit in fork as: meta.fork_pretest_sha +# - "pretest" references sha of latest PR commit as: meta.pr_head_sha +# - "pretest" references sha of latest "subm-pretest" commit as: meta.core_pretest_sha # "pretest" is not updated when only main or custom branch of subm updates # - Otherwise all pretest branches would update at the same time, even on unrelated changes -# "pretest" and "subm-pretest" need to be first checked by a separate "git ls-remote" before "git fetch" -# - Otherwise git fetch would fail fetching them +# "pretest" needs to be first checked by a separate "git ls-remote" before "git fetch" +# - Otherwise git fetch would fail fetching the missing branches # "pretest" and "subm-pretest" can be update manually by push, in case anything new needs to be implemented for the new XML spec # - In that case, there would be no commit sha reference, so it will be re-tested on the next run of this workflow @@ -109,20 +98,27 @@ jobs: $name = $matches[2] $fork_url = "git@github.com:SunSerega/${name}.git" - $merge_sha_by_pr = @{} + $mergeable_prs = @() foreach ($l in Get-GitRemoteBranches $url 'refs/pull/*/merge') { + $pr_num = $l.matches[1] + $mergeable_prs += $pr_num + } + + $head_sha_by_pr = @{} + foreach ($l in Get-GitRemoteBranches $url 'refs/pull/*/head') { $sha = $l.sha $pr_num = $l.matches[1] + if ($pr_num -notin $mergeable_prs) { continue } Write-Host "- Found open PR $pr_num with latest sha $sha" - $merge_sha_by_pr.Add($pr_num, $sha) + $head_sha_by_pr.Add($pr_num, $sha) } - $pretest_sha_by_pr = @{} + $pretest_prs = @{} foreach ($l in Get-GitRemoteBranches $fork_url 'refs/heads/pretest/*') { $sha = $l.sha $pr_num = $l.matches[1] Write-Host "- Found pretest branch for PR $pr_num with latest sha $sha" - $pretest_sha_by_pr.Add($pr_num, $sha) + $pretest_prs += $pr_num } $subm = [PSCustomObject]@{ @@ -131,8 +127,8 @@ jobs: 'owner' = $owner; 'name' = $name; 'repo' = "$owner/$name" - 'merge_sha_by_pr' = $merge_sha_by_pr; - 'pretest_sha_by_pr' = $pretest_sha_by_pr; + 'head_sha_by_pr' = $head_sha_by_pr; + 'pretest_prs' = $pretest_prs; } $submodules += $subm $submodule_by_repo.Add($subm.repo, $subm) @@ -143,19 +139,16 @@ jobs: Write-Host "==============================" Write-Host "Fetching extra data" $core_pretest_pr_nums = @{} - $core_fetch_specs = @() $subm_fetch_specs = @{} foreach ($l in Get-GitRemoteBranches origin 'refs/heads/subm-pretest/*/*/*') { + $sha = $l.sha $subm_owner = $l.matches[1] $subm_name = $l.matches[2] $pr_num = $l.matches[3] $repo = "$subm_owner/$subm_name" Write-Host "- Found core pretest branch for repo $repo PR $pr_num" - $core_pretest_pr_nums[$repo] += @($pr_num) - if ($pr_num -in $submodule_by_repo[$repo].pretest_sha_by_pr.Keys) { - - $core_pretest_branch = "subm-pretest/$repo/$pr_num" - $core_fetch_specs += "refs/heads/${core_pretest_branch}:${core_pretest_branch}" + $core_pretest_pr_nums[$repo] += @{ "$pr_num" = $sha } + if ($pr_num -in $submodule_by_repo[$repo].pretest_prs) { $fork_pretest_branch = "pretest/$pr_num" $subm_fetch_specs[$repo] += @("refs/heads/${fork_pretest_branch}:${fork_pretest_branch}") @@ -166,12 +159,6 @@ jobs: } # To check which pretest branches are up to date and should not be tested # - Only need commit messages, not files - if ($core_fetch_specs) { - Write-Host "- Fetching core branches" - # Write-Host git fetch --depth=1 --no-tags --filter=tree:0 origin $core_fetch_specs - git fetch --depth=1 --no-tags --filter=tree:0 origin $core_fetch_specs - if (-not $?) { throw "git fetch failed" } - } foreach ($repo in $submodule_by_repo.Keys) { if (-not $subm_fetch_specs[$repo]) { continue } Write-Host "- Fetching branches for $repo" @@ -188,35 +175,50 @@ jobs: foreach ($subm in $submodules) { Write-Host "subm: $($subm.name)" - foreach ($pr_num in $subm.merge_sha_by_pr.Keys) { + foreach ($pr_num in $subm.head_sha_by_pr.Keys) { Write-Host "- PR $pr_num" function Is-SkipNeeded() { - if ($pr_num -notin $subm.pretest_sha_by_pr.Keys) { return $false } - if ($pr_num -notin $core_pretest_pr_nums[$subm.repo]) { return $false } + if ($pr_num -notin $subm.pretest_prs) { return $false } + if ($pr_num -notin $core_pretest_pr_nums[$subm.repo].Keys) { return $false } + $extra_debug = $true - function Find-MetaSHA() { - param ($ref, $key) - $pattern = "^meta\.$key=([0-9a-f]{40})$" + $pr_head_sha_found = $false + $core_pretest_sha_found = $false + foreach ($l in git log -1 --pretty=format:"%B" "pretest/$pr_num") { + if ($extra_debug) { + Write-Host "--- $l" + } + if ($l -notmatch '^meta\.(\w+)=([0-9a-f]{40})$') { continue } + $key = $matches[1] + $sha = $matches[2] + if ($extra_debug) { + Write-Host "----- FOUND [$key]: $sha" + } - # Write-Host "--- Looking for commit message lines matching [$pattern]" - $res = @() - foreach ($l in git log -1 --pretty=format:"%B" $ref) { - # Write-Host "----- $l" - if ($l -notmatch $pattern) { continue } - $sha = $matches[1] - # Write-Host "------- FOUND: $sha" - $res += $sha + if ($key -eq 'pr_head_sha') { + + if ($sha -eq $subm.head_sha_by_pr[$pr_num]) { + $pr_head_sha_found = $true + } elseif ($extra_debug) { + Write-Host "----- SHA unexpected" + } + + } elseif ($key -eq 'core_pretest_sha') { + + if ($sha -eq $core_pretest_pr_nums[$subm.repo][$pr_num]) { + $core_pretest_sha_found = $true + } elseif ($extra_debug) { + Write-Host "----- SHA unexpected" + } + + } elseif ($extra_debug) { + Write-Host "----- Key unexpected" } - if (-not $?) { throw "git log failed" } - return $res } - if ($subm.merge_sha_by_pr[$pr_num] -notin (Find-MetaSHA "pretest/$pr_num" 'pr_merge_sha')) { return $false } - if ($subm.pretest_sha_by_pr[$pr_num] -notin (Find-MetaSHA "subm-pretest/$($subm.repo)/$pr_num" 'fork_pretest_sha')) { return $false } - - return $true + return $pr_head_sha_found -and $core_pretest_sha_found } if (Is-SkipNeeded) { continue } @@ -244,17 +246,17 @@ jobs: Write-Host "subm: $($subm.name)" $subm_rem_pretest_pr_nums = @() - foreach ($pr_num in $subm.pretest_sha_by_pr.Keys) { + foreach ($pr_num in $subm.pretest_prs) { Write-Host "- Fork pretest for PR $pr_num" - if ($pr_num -in $subm.merge_sha_by_pr.Keys) { continue } + if ($pr_num -in $subm.pr_head_sha.Keys) { continue } $subm_rem_pretest_pr_nums += $pr_num Write-Host "--- REMOVE" } $core_rem_pretest_pr_nums = @() - foreach ($pr_num in $core_pretest_pr_nums[$subm.repo]) { + foreach ($pr_num in $core_pretest_pr_nums[$subm.repo].Keys) { Write-Host "- Core pretest for PR $pr_num" - if ($pr_num -in $subm.merge_sha_by_pr.Keys) { continue } + if ($pr_num -in $subm.pr_head_sha.Keys) { continue } $core_rem_pretest_pr_nums += $pr_num } @@ -309,22 +311,6 @@ jobs: repository: ${{ matrix.exec-data.fork_repo }} token: ${{ secrets.POCGL_pretest_upstream_PAT }} - - name: Install OpenCL driver for Intel CPU - run: | - - # https://www.intel.com/content/www/us/en/developer/articles/technical/intel-cpu-runtime-for-opencl-applications-with-sycl-support.html - Invoke-WebRequest -Uri 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0e6849e6-2c56-480b-afcf-be8331d5c4f6-opencl/w_opencl_runtime_p_2024.1.0.968.exe' -OutFile 'D:\igfx.exe' - - 7z x "D:\igfx.exe" -o"D:\igfx" -y - - D:\igfx\w_opencl_runtime_p_2024.1.0.968.msi /quiet - - - name: Download and unpack Pascal compiler - run: | - - Invoke-WebRequest -Uri 'https://github.com/SunSerega/pascalabcnet/releases/download/custom-build-tag/PABCNETC.zip' -OutFile 'D:\PABCNETC.zip' - Expand-Archive -Path 'D:\PABCNETC.zip' -DestinationPath 'D:\PABCNETC' -Force - - name: (!) pretest run: | @@ -335,70 +321,102 @@ jobs: + $remote_official = '0_official' + + $b_fork_upstream_main = "$remote_official/main" + $b_fork_main = 'custom' + $b_fork_pretest = "pretest/$pr_num" + + $b_fork_merged_pr_head = 'merged_pr_head' # $pr_num/head + main + $b_fork_virgin_merge_test = 'virgin_merge_test' # $b_merged_pr_head + custom + + $b_core_main = '${{ github.event.repository.default_branch }}' + $b_core_subm_pretest = "subm-pretest/$org_repo/$pr_num" + + + Write-Host "Updating subm fork..." Push-Location './fork' - git remote add 0_official "git@github.com:${org_repo}.git" - git config --add remote.0_official.fetch '+refs/pull/*:refs/remotes/0_official/pull/*' + git remote add $remote_official "git@github.com:${org_repo}.git" + git config --add "remote.$remote_official.fetch" "+refs/pull/*:refs/remotes/$remote_official/pull/*" git fetch --all 2>&1 | Out-Null - $fork_main_branch_name = 'custom' - $fork_branch_name = "pretest/$pr_num" - if (&{ git show-ref --verify -q "refs/remotes/origin/$fork_branch_name"; $? }) { - Write-Host "Branch for PR $pr_num exists" - git checkout $fork_branch_name - - Write-Host "Merging latest commits from PR $pr_num" - git merge "remotes/0_official/pull/$pr_num/merge" - if (-not $?) { throw "git merge failed" } - git push - if (-not $?) { throw "git push failed" } - - } else { - Write-Host "Branch for PR $pr_num doesn't exist" - git checkout "remotes/0_official/pull/$pr_num/merge" - if (-not $?) { throw "git checkout failed" } - - Write-Host "Creating:" - git checkout -b $fork_branch_name - if (-not $?) { throw "git checkout -b failed" } - git push --set-upstream origin $fork_branch_name - if (-not $?) { throw "git push failed" } - - } + Write-Host "- Creating $b_fork_merged_pr_head" + git checkout "remotes/$remote_official/pull/$pr_num/head" + if (-not $?) { throw "git checkout failed" } + git checkout -b $b_fork_merged_pr_head + if (-not $?) { throw "git checkout -b failed" } - Write-Host "Merging with 0_official/main:" - git merge 0_official/main + Write-Host "--- Merging with ${b_fork_upstream_main}:" + git merge $b_fork_upstream_main if (-not $?) { Write-Host "git merge failed" Write-Host "This PR is outdated and cannot be properly tested" exit 0 } - git push - if (-not $?) { throw "git push failed" } - Write-Host "Merging with ${fork_main_branch_name}:" - $fork_pretest_old_sha = git rev-parse HEAD - $pr_merge_sha = git rev-parse remotes/0_official/pull/$pr_num/merge - git merge $fork_main_branch_name --no-ff -m @" - Merge $fork_main_branch_name into $fork_branch_name - meta.pr_merge_sha=$pr_merge_sha - "@ - if (-not $?) { throw "git merge failed" } - git push - if (-not $?) { throw "git push failed" } + Write-Host "- Creating $b_fork_virgin_merge_test" + git checkout -b $b_fork_virgin_merge_test + if (-not $?) { throw "git checkout -b failed" } + $virgin_merge_test_exists = $true - $fork_pretest_sha = git rev-parse HEAD - $empty_pretest_message = @" - Update pretest meta - meta.pr_merge_sha=$pr_merge_sha - "@ - if ($fork_pretest_sha -eq $fork_pretest_old_sha) { - Write-Host "Merge with ${fork_main_branch_name} was skipped, adding an empty meta commit" - git commit -a --allow-empty -m $empty_pretest_message - if (-not $?) { throw "git commit failed" } - git push + Write-Host "--- Merging with ${b_fork_main}:" + git merge $b_fork_main + if (-not $?) { + Write-Host "git merge $b_fork_main failed" + Write-Host "Cannot create $b_fork_virgin_merge_test" + $virgin_merge_test_exists = $false + } + + if (&{ git show-ref --verify -q "refs/remotes/origin/$b_fork_pretest"; $? }) { + Write-Host "- Branch for PR $pr_num exists" + git checkout $b_fork_pretest + + Write-Host "--- Merging with ${b_fork_merged_pr_head}:" + git merge $b_fork_merged_pr_head + if (-not $?) { throw "git merge failed" } + + Write-Host "--- Merging with ${b_fork_main}:" + git merge $b_fork_main + if (-not $?) { throw "git merge failed" } + + $need_push = $true + if ($virgin_merge_test_exists) { + Write-Host "--- Diff with ${b_fork_virgin_merge_test}:" + git diff HEAD $b_fork_virgin_merge_test --exit-code + if (-not $?) { + Write-Host "----- Diff found, building on top" + } else { + Write-Host "----- Diff not found, resetting branch" + git reset --hard $b_fork_virgin_merge_test + if (-not $?) { throw "git reset failed" } + git push -f + if (-not $?) { throw "git push -f failed" } + $need_push = $false + } + } + + if ($need_push) { + git push + if (-not $?) { throw "git push failed" } + } + + } else { + Write-Host "- Branch for PR $pr_num doesn't exist" + + if (-not $virgin_merge_test_exists) { + throw "Cannot do anything without $b_fork_virgin_merge_test" + } + + Write-Host "--- Creating:" + git checkout $b_fork_virgin_merge_test + if (-not $?) { throw "git checkout failed" } + git checkout -b $b_fork_pretest + if (-not $?) { throw "git checkout -b failed" } + + git push --set-upstream origin $b_fork_pretest if (-not $?) { throw "git push failed" } - $fork_pretest_sha = git rev-parse HEAD + } Pop-Location @@ -409,41 +427,104 @@ jobs: Push-Location './core' git fetch --all 2>&1 | Out-Null - $core_main_branch_name = '${{ github.event.repository.default_branch }}' - $core_branch_name = "subm-pretest/$org_repo/$pr_num" - if (&{ git show-ref --verify -q "refs/remotes/origin/$core_branch_name"; $? }) { - Write-Host "Branch for PR $pr_num exists. Merging with ${core_main_branch_name}:" - git checkout $core_branch_name + if (&{ git show-ref --verify -q "refs/remotes/origin/$b_core_subm_pretest"; $? }) { + Write-Host "- Branch for PR $pr_num exists" + git checkout $b_core_subm_pretest if (-not $?) { throw "git checkout failed" } - git merge $core_main_branch_name -m "[trivial] Merge $core_main_branch_name into subm-pretest/... branch" + + Write-Host "--- Deleting the latest [trivial] commits:" + while ($true) { + ($commit_name = (git log -1 --pretty=format:"%B" HEAD) -join "`n") + if (-not $?) { throw "git log failed" } + if (-not $commit_name.Contains("[trivial]")) { break } + Write-Host "=====" + git reset --hard HEAD~1 + if (-not $?) { throw "git reset failed" } + } + + # expecting fast-forward when possible + Write-Host "--- Merging with ${b_core_main}:" + git merge $b_core_main -m "[trivial] Merge $b_core_main into subm-pretest/..." if (-not $?) { throw "git merge failed" } git push if (-not $?) { throw "git push failed" } + } else { - Write-Host "Branch for PR $pr_num doesn't exist. Creating:" - git checkout -b $core_branch_name + Write-Host "- Branch for PR $pr_num doesn't exist" + + Write-Host "--- Creating:" + git checkout -b $b_core_subm_pretest if (-not $?) { throw "git checkout -b failed" } - git push --set-upstream origin $core_branch_name + git push --set-upstream origin $b_core_subm_pretest if (-not $?) { throw "git push failed" } + } - & .\DeleteAllTemp.bat NoPause | Out-Null + $core_pretest_sha = git rev-parse HEAD + Pop-Location - Write-Host "Compile: " -NoNewline - Start-Process -FilePath 'D:\PABCNETC\pabcnetcclear.exe' -ArgumentList '"PackAll.pas"' -Wait -NoNewWindow - & .\PackAll.exe "Stages= PullUpstream + Reference + Dummy + OpenCL+OpenCLABC + OpenGL+OpenGLABC + Compile + Test + Release" "PullUpstreamBranch=${subm_name}:${fork_branch_name}" "PasCompPath=D:\PABCNETC\pabcnetcclear.exe" SkipFinishedPause - if (-not $?) { throw "PackAll failed" } - git commit -a --allow-empty -m @" - [trivial] test changes - meta.fork_pretest_sha=$fork_pretest_sha + Write-Host "Adding meta to fork repo..." + Push-Location './fork' + + git commit --allow-empty -m @" + Add pretest meta + meta.pr_head_sha=$pr_head_sha + meta.core_pretest_sha=$core_pretest_sha "@ if (-not $?) { throw "git commit failed" } git push if (-not $?) { throw "git push failed" } Pop-Location + + + + & { + Write-Host "Installing OpenCL driver for Intel CPU" + + # https://www.intel.com/content/www/us/en/developer/articles/technical/intel-cpu-runtime-for-opencl-applications-with-sycl-support.html + Invoke-WebRequest -Uri 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0e6849e6-2c56-480b-afcf-be8331d5c4f6-opencl/w_opencl_runtime_p_2024.1.0.968.exe' -OutFile 'D:\igfx.exe' + + 7z x "D:\igfx.exe" -o"D:\igfx" -y + + D:\igfx\w_opencl_runtime_p_2024.1.0.968.msi /quiet + } + + & { + Write-Host "Downloading and unpacking Pascal compiler" + + Invoke-WebRequest -Uri 'https://github.com/SunSerega/pascalabcnet/releases/download/custom-build-tag/PABCNETC.zip' -OutFile 'D:\PABCNETC.zip' + + Expand-Archive -Path 'D:\PABCNETC.zip' -DestinationPath 'D:\PABCNETC' -Force + } + + + + Write-Host "Testing main repo..." + Push-Location './core' + + & .\DeleteAllTemp.bat NoPause | Out-Null + + Write-Host "- Compile: " -NoNewline + Start-Process -FilePath 'D:\PABCNETC\pabcnetcclear.exe' -ArgumentList '"PackAll.pas"' -Wait -NoNewWindow + + & .\PackAll.exe "Stages= PullUpstream + Reference + Dummy + OpenCL+OpenCLABC + OpenGL+OpenGLABC + Compile + Test + Release" "PullUpstreamBranch=${subm_name}:${b_fork_pretest}" "PasCompPath=D:\PABCNETC\pabcnetcclear.exe" SkipFinishedPause + if (-not $?) { throw "PackAll failed" } + + $need_commit = (git diff --name-only ":!LastPack.log" ":!Log/PullUpstream.log" ":!DataScraping/Reps") -or (git ls-files --others --exclude-standard) + if ($need_commit) { + Write-Host "- Found significant changes" + + git commit -a -m '[trivial] test changes' + if (-not $?) { throw "git commit failed" } + git push + if (-not $?) { throw "git push failed" } + + } + + Pop-Location remove-old-pretest: runs-on: windows-latest