better benchmark structure

.github/workflows/benchmark.yml

@@ -1,29 +1,79 @@
-name: Run benchmarks
+name: Benchmark a pull request
 
 on:
-  pull_request:
-    types: [labeled, opened, synchronize, reopened]
+  pull_request_target:
+    branches:
+      - main
+
+permissions:
+  pull-requests: write
 
-# Only trigger the benchmark job when you add `run benchmark` label to the PR
 jobs:
-  Benchmark:
-    runs-on: ubuntu-latest
-    if: contains(github.event.pull_request.labels.*.name, 'run benchmark')
-    steps:
-      - uses: actions/checkout@v2
-      - uses: julia-actions/setup-julia@latest
-        with:
-          version: 1
-      - uses: julia-actions/cache@v1
-      - name: Install registered dependencies
-        uses: julia-actions/julia-buildpkg@latest
-      - name: Install dependencies
-        run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI"'
-      - name: Run benchmarks
-        run: julia -e 'using BenchmarkCI; BenchmarkCI.judge(baseline="origin/main")'
-      - name: check git
-        run: julia -e 'read(run(`git status`))'
-      - name: Post results
-        run: julia --project=. -e 'using BenchmarkCI; BenchmarkCI.postjudge()'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    generate_plots:
+        runs-on: ubuntu-latest
+        if: contains(github.event.pull_request.labels.*.name, 'run benchmark')
+
+        steps:
+            - uses: actions/checkout@v2
+            - uses: julia-actions/setup-julia@v1
+              with:
+                version: "1"
+            - uses: julia-actions/cache@v1
+            - name: Extract Package Name from Project.toml
+              id: extract-package-name
+              run: |
+                PACKAGE_NAME=$(grep "^name" Project.toml | sed 's/^name = "\(.*\)"$/\1/')
+                echo "::set-output name=package_name::$PACKAGE_NAME"
+            - name: Build AirspeedVelocity
+              env:
+                JULIA_NUM_THREADS: 2
+              run: |
+                # Lightweight build step, as sometimes the runner runs out of memory:
+                julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.add(;url="https://github.com/MilesCranmer/AirspeedVelocity.jl.git")'
+                julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.build("AirspeedVelocity")'
+            - name: Add ~/.julia/bin to PATH
+              run: |
+                echo "$HOME/.julia/bin" >> $GITHUB_PATH
+            - name: Run benchmarks
+              run: |
+                echo $PATH
+                ls -l ~/.julia/bin
+                mkdir results
+                benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.repository.default_branch}}" --output-dir=results/ --tune
+            - name: Create plots from benchmarks
+              run: |
+                mkdir -p plots
+                benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/
+            - name: Upload plot as artifact
+              uses: actions/upload-artifact@v2
+              with:
+                name: plots
+                path: plots
+            - name: Create markdown table from benchmarks
+              run: |
+                benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md
+                echo '### Benchmark Results' > body.md
+                echo '' >> body.md
+                echo '' >> body.md
+                cat table.md >> body.md
+                echo '' >> body.md
+                echo '' >> body.md
+                echo '### Benchmark Plots' >> body.md
+                echo 'A plot of the benchmark results have been uploaded as an artifact to the workflow run for this PR.' >> body.md
+                echo 'Go to "Actions"->"Benchmark a pull request"->[the most recent run]->"Artifacts" (at the bottom).' >> body.md
+
+            - name: Find Comment
+              uses: peter-evans/find-comment@v2
+              id: fcbenchmark
+              with:
+                issue-number: ${{ github.event.pull_request.number }}
+                comment-author: 'github-actions[bot]'
+                body-includes: Benchmark Results
+
+            - name: Comment on PR
+              uses: peter-evans/create-or-update-comment@v3
+              with:
+                comment-id: ${{ steps.fcbenchmark.outputs.comment-id }}
+                issue-number: ${{ github.event.pull_request.number }}
+                body-path: body.md
+                edit-mode: replace

.github/workflows/documenter.yml

@@ -11,6 +11,7 @@ jobs:
   Documenter:
     name: Documentation
     runs-on: ubuntu-latest
+    if: "!(contains(github.event.pull_request.labels.*.name, 'dont run documenter'))"
     steps:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1

.github/workflows/unittests.yml

@@ -20,6 +20,7 @@ on:
 jobs:
   build:
     runs-on: ${{ matrix.os }}
+    if: "!(contains(github.event.pull_request.labels.*.name, 'dont run tests'))"
     strategy:
       matrix:
         julia-version: [1] # 1 without specifier uses the current release