chore: leaf aggregation tuning (#1150) #2011
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "OpenVM Benchmarks: Coordinate Runner & Reporting" | |
| on: | |
| push: | |
| branches: ["main"] | |
| pull_request: | |
| types: [opened, synchronize, reopened, labeled] | |
| branches: ["**"] | |
| paths: | |
| - "crates/circuits/**" | |
| - "crates/vm/**" | |
| - "crates/toolchain/**" | |
| - "crates/extensions/**" | |
| - "benchmarks/**" | |
| - ".github/workflows/benchmark-call.yml" | |
| - ".github/workflows/benchmarks.yml" | |
| concurrency: | |
| group: benchmark-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| OPENVM_FAST_TEST: "1" | |
| CURRENT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
| REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| create-matrix: | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - runner=8cpu-linux-x64 | |
| outputs: | |
| matrix: ${{ steps.create-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.CURRENT_SHA }} | |
| repository: ${{ env.REPO }} | |
| - name: Create benchmark matrix from JSON | |
| id: create-matrix | |
| run: | | |
| if [ ! -f ./ci/benchmark-config.json ]; then | |
| echo "Error: ./ci/benchmark-config.json not found" | |
| exit 1 | |
| fi | |
| cat ./ci/benchmark-config.json | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| LABELS='${{ toJson(github.event.pull_request.labels) }}' | |
| RUN_E2E=$(echo "$LABELS" | jq 'any(.name == "run-benchmark-e2e")') | |
| else | |
| RUN_E2E=false | |
| fi | |
| matrix=$(jq -c --argjson run_e2e $RUN_E2E ' | |
| [ | |
| .benchmarks[] | | |
| .name as $name | | |
| .id as $id | | |
| .e2e_bench as $e2e_bench | | |
| select($run_e2e or .e2e_bench != true) | | |
| .run_params[] | | |
| { | |
| name: $name, | |
| e2e_bench: $e2e_bench, | |
| id: $id, | |
| instance_type: .instance_type, | |
| memory_allocator: .memory_allocator, | |
| app_log_blowup: .app_log_blowup, | |
| agg_log_blowup: .agg_log_blowup, | |
| root_log_blowup: (.root_log_blowup // 0), | |
| internal_log_blowup: (.internal_log_blowup // 0), | |
| max_segment_length: (.max_segment_length // 1048476) | |
| } | |
| ] | |
| ' ./ci/benchmark-config.json) | |
| if [ $? -ne 0 ]; then | |
| echo "Error: Failed to parse ./ci/benchmark-config.json" | |
| exit 1 | |
| fi | |
| echo "matrix=$matrix" >> $GITHUB_OUTPUT | |
| benchmark: | |
| needs: create-matrix | |
| strategy: | |
| matrix: | |
| benchmark_run: ${{fromJson(needs.create-matrix.outputs.matrix)}} | |
| uses: ./.github/workflows/benchmark-call.yml | |
| with: | |
| benchmark_name: ${{ matrix.benchmark_run.name }} | |
| benchmark_id: ${{ matrix.benchmark_run.id }} | |
| instance_type: ${{ matrix.benchmark_run.instance_type }} | |
| memory_allocator: ${{ matrix.benchmark_run.memory_allocator }} | |
| app_log_blowup: ${{ matrix.benchmark_run.app_log_blowup }} | |
| agg_log_blowup: ${{ matrix.benchmark_run.agg_log_blowup }} | |
| root_log_blowup: ${{ matrix.benchmark_run.root_log_blowup }} | |
| internal_log_blowup: ${{ matrix.benchmark_run.internal_log_blowup }} | |
| max_segment_length: ${{ matrix.benchmark_run.max_segment_length }} | |
| e2e_bench: ${{ matrix.benchmark_run.e2e_bench }} | |
| secrets: inherit | |
| summarize: | |
| needs: [create-matrix, benchmark] | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - runner=8cpu-linux-arm64 | |
| steps: | |
| ########################################################################## | |
| # Install S3 if necessary # | |
| ########################################################################## | |
| - name: Install architecture specific tools | |
| run: | | |
| S5CMD_BIN="s5cmd_2.2.2_linux_arm64.deb" | |
| echo "Checking s5cmd" | |
| if type s5cmd &>/dev/null; then | |
| echo "s5cmd was installed." | |
| else | |
| TMP_DIR=/tmp/s5cmd | |
| rm -rf $TMP_DIR | |
| mkdir $TMP_DIR | |
| echo "s5cmd was not installed. Installing.." | |
| wget "https://github.com/peak/s5cmd/releases/download/v2.2.2/${S5CMD_BIN}" -P $TMP_DIR | |
| sudo dpkg -i "${TMP_DIR}/${S5CMD_BIN}" | |
| fi | |
| ########################################################################## | |
| # Download individual result .md files from S3 and combine them # | |
| ########################################################################## | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.CURRENT_SHA }} | |
| repository: ${{ env.REPO }} | |
| - name: Set github pages path for PR | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| BENCHMARK_RESULTS_PATH="benchmarks-pr/${{ github.event.pull_request.number }}" | |
| echo "BENCHMARK_RESULTS_PATH=${BENCHMARK_RESULTS_PATH}" >> $GITHUB_ENV | |
| - name: Set github pages path for push | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| run: | | |
| BENCHMARK_RESULTS_PATH="benchmarks" | |
| echo "BENCHMARK_RESULTS_PATH=${BENCHMARK_RESULTS_PATH}" >> $GITHUB_ENV | |
| - name: Load all metadata files from S3 | |
| run: | | |
| current_sha=$(git rev-parse HEAD) | |
| md_files=$(echo '${{ needs.create-matrix.outputs.matrix }}' | jq -r ' | |
| .[] | | |
| select(.e2e_bench != true) | | |
| "\(.id)-"' | | |
| sed "s/$/${current_sha}.md/" | | |
| sort) | |
| md_file_list=$(echo -n "$md_files" | paste -sd "," -) | |
| e2e_md_files=$(echo '${{ needs.create-matrix.outputs.matrix }}' | jq -r ' | |
| .[] | | |
| select(.e2e_bench == true) | | |
| "\(.id)-"' | | |
| sed "s/$/${current_sha}.md/" | | |
| sort) | |
| e2e_md_file_list=$(echo -n "$e2e_md_files" | paste -sd "," -) | |
| while read md_file; do | |
| if [ -z "$md_file" ]; then | |
| continue | |
| fi | |
| echo "Downloading results for benchmark: $md_file" | |
| s5cmd cp "s3://openvm-public-data-sandbox-us-east-1/benchmark/github/results/${md_file}" "${md_file}" | |
| done <<< "$md_files" | |
| E2E_FILE_LIST="" | |
| if [[ -n "$e2e_md_files" ]]; then | |
| while read e2e_md_file; do | |
| echo "Downloading results for benchmark: $e2e_md_file" | |
| s5cmd cp "s3://openvm-public-data-sandbox-us-east-1/benchmark/github/results/${e2e_md_file}" "${e2e_md_file}" | |
| done <<< "$e2e_md_files" | |
| E2E_FILE_LIST="${e2e_md_file_list}" | |
| fi | |
| echo "Benchmarks: ${md_file_list}" | |
| echo "E2E Benchmarks: ${E2E_FILE_LIST}" | |
| python3 ci/scripts/metric_unify/summarize.py "${md_file_list}" \ | |
| --e2e-md-files "${E2E_FILE_LIST}" \ | |
| --aggregation-json ci/scripts/metric_unify/aggregation.json \ | |
| --benchmark-results-link "https://github.com/${{ github.repository }}/blob/benchmark-results/${BENCHMARK_RESULTS_PATH}" | |
| echo "" >> summary.md | |
| echo "Commit: https://github.com/${{ github.repository }}/commit/${CURRENT_SHA}" >> summary.md | |
| echo "" >> summary.md | |
| echo "[Benchmark Workflow](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> summary.md | |
| mkdir -p /tmp/benchmark-results/ | |
| cp summary.md /tmp/benchmark-results/ | |
| ########################################################################## | |
| # Update benchmark-results with summary upon a PR event # | |
| ########################################################################## | |
| - uses: actions/checkout@v4 | |
| if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main') | |
| with: | |
| ref: benchmark-results | |
| - name: Set up git | |
| if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main') | |
| run: | | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --global user.name "github-actions[bot]" | |
| - name: Update github pages with new bench results | |
| if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event_name == 'push' && github.ref == 'refs/heads/main') | |
| run: | | |
| mkdir -p ${BENCHMARK_RESULTS_PATH} | |
| cp /tmp/benchmark-results/summary.md ${BENCHMARK_RESULTS_PATH}/summary.md | |
| git add ${BENCHMARK_RESULTS_PATH}/summary.md | |
| if [[ "${{ github.event_name }}" == "push" ]]; then | |
| cp /tmp/benchmark-results/summary.md index.md | |
| git add index.md | |
| fi | |
| git commit --allow-empty -m "Update summarized benchmark result at ${BENCHMARK_RESULTS_PATH}/summary.md" | |
| git push --force | |
| ########################################################################## | |
| # Update PR comment upon a pull request event # | |
| ########################################################################## | |
| - name: Collapse previous comment (if exists) | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const comments = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number | |
| }); | |
| for (const comment of comments.data) { | |
| if (comment.user.login == "github-actions[bot]" && comment.body.startsWith("<!--Benchmark Results-->")) { | |
| console.log("collapse comment ", comment.id); | |
| const resp = await github.graphql(` | |
| mutation { | |
| minimizeComment(input: {classifier: OUTDATED, subjectId: "${comment.node_id}"}) { | |
| minimizedComment { | |
| isMinimized | |
| } | |
| } | |
| } | |
| `); | |
| } | |
| } | |
| - name: Add comment to pull request | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs') | |
| const newBenchmark = fs.readFileSync('/tmp/benchmark-results/summary.md', { encoding: 'utf8', flag: 'r' }) | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: `<!--Benchmark Results-->\n${newBenchmark}` | |
| }); |