Benchmark: running individual benchmarks #14
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Benchmark: Coordinate Runner & Reporting" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| benchmark_name: | |
| type: choice | |
| required: true | |
| description: The name of the benchmark to run | |
| options: | |
| - verify_fibair | |
| - fibonacci | |
| - alu256_e2e | |
| - small_e2e | |
| instance_type: | |
| type: string | |
| required: false | |
| description: The type of runner to start ({1,2,4,8,16,32,48,64}cpu-linux-arm64) | |
| default: 64cpu-linux-arm64 | |
| workflow_call: | |
| inputs: | |
| benchmark_name: | |
| type: string | |
| required: true | |
| description: The name of the benchmark to run | |
| instance_type: | |
| type: string | |
| required: false | |
| description: The type of runner to start ({1,2,4,8,16,32,48,64}cpu-linux-arm64) | |
| default: 64cpu-linux-arm64 | |
| env: | |
| S3_PATH: s3://axiom-workflow-data-sandbox-us-east-1/benchmark/github/results | |
| S3_METRICS_PATH: s3://axiom-workflow-data-sandbox-us-east-1/benchmark/github/metrics | |
| PUBLIC_S3_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/flamegraphs | |
| FEATURE_FLAGS: "bench-metrics,parallel,mimalloc" | |
| CMD_ARGS: "" | |
| jobs: | |
| bench-new: | |
| name: Run benchmark on workflow ref/branch | |
| runs-on: | |
| - runs-on | |
| - runner=${{ inputs.instance_type }} | |
| - image=arm64-rust-dev-32gb | |
| - run-id=${{ github.run_id }} | |
| - tag=bench-${{ inputs.benchmark_name }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref || github.ref }} | |
| - uses: dtolnay/rust-toolchain@nightly | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| cache-on-failure: true | |
| - name: Install rust src | |
| run: | | |
| rustup component add rust-src --toolchain nightly-aarch64-unknown-linux-gnu | |
| - name: Feature flags | |
| if: contains(github.event.pull_request.labels.*.name, 'run-benchmark') || (github.event_name == 'push' && github.ref == 'refs/heads/main') | |
| run: | | |
| echo "Adding aggregation feature flag" | |
| echo "FEATURE_FLAGS=${FEATURE_FLAGS},aggregation" >> $GITHUB_ENV | |
| - name: Setup halo2 | |
| working-directory: lib/recursion # We only ever run halo2 for recursion | |
| run: | | |
| if [[ "${{ github.event.pull_request.labels[*].name }}" == *"run-benchmark-e2e"* ]]; then | |
| echo "Adding static-verifier feature flag" | |
| echo "FEATURE_FLAGS=${FEATURE_FLAGS},static-verifier" >> $GITHUB_ENV | |
| bash trusted_setup_s3.sh | |
| fi | |
| - name: Set BIN_NAME and CMD_ARGS | |
| run: | | |
| CMD_ARGS="--features ${FEATURE_FLAGS}" | |
| echo "CMD_ARGS=${CMD_ARGS}" >> $GITHUB_ENV | |
| echo "BIN_NAME=${{ inputs.benchmark_name }}" >> $GITHUB_ENV | |
| ###################################################### | |
| # Run a different benchmark based on benchmark_name: | |
| - name: Run benchmark | |
| if: inputs.benchmark_name == 'verify_fibair' | |
| working-directory: lib/recursion | |
| run: | | |
| python3 ../../ci/scripts/bench.py $BIN_NAME $CMD_ARGS | |
| - name: Run benchmark | |
| if: inputs.benchmark_name == 'fibonacci' | |
| working-directory: benchmarks | |
| run: | | |
| python3 ../ci/scripts/bench.py $BIN_NAME $CMD_ARGS | |
| - name: Run benchmark | |
| if: inputs.benchmark_name == 'alu256_e2e' | |
| working-directory: lib/recursion | |
| run: | | |
| python3 ../../ci/scripts/bench.py $BIN_NAME $CMD_ARGS | |
| - name: Run benchmark | |
| if: inputs.benchmark_name == 'small_e2e' | |
| working-directory: lib/recursion | |
| run: | | |
| python3 ../../ci/scripts/bench.py $BIN_NAME $CMD_ARGS | |
| ###################################################### | |
| - name: Store metric json and compute diff with previous | |
| run: | | |
| METRIC_PATH=".bench_metrics/${BIN_NAME}.json" | |
| echo "METRIC_PATH=${METRIC_PATH}" >> $GITHUB_ENV | |
| current_sha=$(git rev-parse HEAD) | |
| echo "Current SHA: $current_sha" | |
| echo "current_sha=${current_sha}" >> $GITHUB_ENV | |
| if [[ -f $METRIC_PATH ]]; then | |
| s5cmd cp $METRIC_PATH ${{ env.S3_METRICS_PATH }}/${current_sha}-${{ inputs.benchmark_name }}.json | |
| prev_path="${{ env.S3_METRICS_PATH }}/main-${{ inputs.benchmark_name }}.json" | |
| count=`s5cmd ls $prev_path | wc -l` | |
| if [[ $count -gt 0 ]]; then | |
| s5cmd cp $prev_path prev.json | |
| python3 ci/scripts/metric_unify/main.py $METRIC_PATH --prev prev.json --aggregation-json ci/scripts/metric_unify/aggregation.json > results.md | |
| else | |
| echo "No previous benchmark on main branch found" | |
| python3 ci/scripts/metric_unify/main.py $METRIC_PATH --aggregation-json ci/scripts/metric_unify/aggregation.json > results.md | |
| fi | |
| else | |
| echo "No benchmark metrics found at ${METRIC_PATH}" | |
| fi | |
| - name: Install inferno-flamegraph | |
| run: cargo install inferno | |
| - name: Generate flamegraphs | |
| run: | | |
| if [[ -f $METRIC_PATH ]]; then | |
| python3 ci/scripts/metric_unify/flamegraph.py $METRIC_PATH | |
| s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.PUBLIC_S3_PATH }}/${current_sha}/" | |
| echo "UPLOAD_FLAMEGRAPHS=1" >> $GITHUB_ENV | |
| fi | |
| - name: Add benchmark metadata | |
| run: | | |
| commit_url="https://github.com/${{ github.repository }}/commit/${current_sha}" | |
| RESULT_PATH=results.md | |
| echo "" >> $RESULT_PATH | |
| if [[ "$UPLOAD_FLAMEGRAPHS" == '1' ]]; then | |
| echo "<details>" >> $RESULT_PATH | |
| echo "<summary>Flamegraphs</summary>" >> $RESULT_PATH | |
| echo "" >> $RESULT_PATH | |
| for file in .bench_metrics/flamegraphs/*.svg; do | |
| filename=$(basename "$file") | |
| flamegraph_url=https://axiom-public-data-sandbox-us-east-1.s3.us-east-1.amazonaws.com/benchmark/github/flamegraphs/${current_sha}/${filename} | |
| echo "[]($flamegraph_url)" >> $RESULT_PATH | |
| done | |
| echo "" >> $RESULT_PATH | |
| echo "</details>" >> $RESULT_PATH | |
| echo "" >> $RESULT_PATH | |
| fi | |
| echo "Commit: ${commit_url}" >> $RESULT_PATH | |
| echo "Instance Type: ${{ inputs.instance_type }}" >> $RESULT_PATH | |
| echo "[Benchmark Workflow](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $RESULT_PATH | |
| s5cmd cp $RESULT_PATH "${{ env.S3_PATH }}/${current_sha}-${{ inputs.benchmark_name }}.md" | |
| - name: Collapse previous comment (if exists) | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const comments = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number | |
| }); | |
| for (const comment of comments.data) { | |
| if (comment.user.login == "github-actions[bot]" && comment.body.startsWith("<!--${{ inputs.benchmark_name }}-->")) { | |
| console.log("collapse comment ", comment.id); | |
| const resp = await github.graphql(` | |
| mutation { | |
| minimizeComment(input: {classifier: OUTDATED, subjectId: "${comment.node_id}"}) { | |
| minimizedComment { | |
| isMinimized | |
| } | |
| } | |
| } | |
| `); | |
| } | |
| } | |
| - name: Add comment to pull request | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs') | |
| const newBenchmark = fs.readFileSync('./results.md', { encoding: 'utf8', flag: 'r' }) | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: `<!--${{ inputs.benchmark_name }}-->\n${newBenchmark}` | |
| }); | |
| - name: Update latest main result in s3 | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| run: | | |
| s5cmd cp "${{ env.S3_PATH }}/${{ env.current_sha }}-${{ inputs.benchmark_name }}.md" "${{ env.S3_PATH }}/main-${{ inputs.benchmark_name }}.md" | |
| if [[ -f $METRIC_PATH ]]; then | |
| s5cmd cp $METRIC_PATH "${{ env.S3_METRICS_PATH }}/main-${{ inputs.benchmark_name }}.json" | |
| fi | |
| - uses: actions/checkout@v4 | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| with: | |
| ref: gh-pages | |
| - name: set up git | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| run: | | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --global user.name "github-actions[bot]" | |
| - name: Update github pages with new bench results | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| run: | | |
| s5cmd cp "${{ env.S3_PATH }}/main-${{ inputs.benchmark_name }}.md" benchmarks/${{ inputs.benchmark_name }}.md | |
| git add benchmarks/${{ inputs.benchmark_name }}.md | |
| git commit --allow-empty -m "Update benchmark result for ${{ inputs.benchmark_name }}" | |
| git push |