benchmark-call.yml

name: "Benchmark: running individual benchmarks"

on:
  workflow_dispatch:
    inputs:
      benchmark_name:
        type: choice
        required: true
        description: The name of the benchmark to run
        options:
          - verify_fibair
          - fibonacci
          - revm_transfer
          - regex
          - base64_json
          - fib_e2e
      instance_type:
        type: string
        required: false
        description: The type of runner to start ({1,2,4,8,16,32,48,64}cpu-linux-{arm64,x64})
        default: 64cpu-linux-arm64
      memory_allocator:
        type: string
        required: false
        description: Memory allocator to use (mimalloc or jemalloc)
        default: mimalloc
      app_log_blowup:
        type: number
        required: false
        description: Application level log blowup
        default: 2
      agg_log_blowup:
        type: number
        required: false
        description: Aggregation (leaf) level log blowup
        default: 2
      root_log_blowup:
        type: number
        required: false
        description: Root level log blowup (only for e2e)
        default: 2
      internal_log_blowup:
        type: number
        required: false
        description: Internal level log blowup (only for e2e)
        default: 2
      max_segment_length:
        type: number
        required: false
        description: Max segment length for continuations, must be larger than 524288
        default: 1048476
      e2e_bench:
        type: boolean
        required: true
        description: Whether to run the e2e benchmark
  workflow_call:
    inputs:
      benchmark_name:
        type: string
        required: true
        description: The name of the benchmark to run
      benchmark_id:
        type: string
        required: true
        description: The id of the benchmark to run, must be unique within matrix
      instance_type:
        type: string
        required: false
        description: The type of runner to start ({1,2,4,8,16,32,48,64}cpu-linux-{arm64,x64})
        default: 64cpu-linux-arm64
      memory_allocator:
        type: string
        required: false
        description: Memory allocator to use (mimalloc or jemalloc)
        default: mimalloc
      app_log_blowup:
        type: number
        required: false
        description: Application level log blowup
        default: 2
      agg_log_blowup:
        type: number
        required: false
        description: Aggregation (leaf) level log blowup
        default: 2
      root_log_blowup:
        type: number
        required: false
        description: Root level log blowup (only for e2e)
        default: 2
      internal_log_blowup:
        type: number
        required: false
        description: Internal level log blowup (only for e2e)
        default: 2
      max_segment_length:
        type: number
        required: false
        description: Max segment length for continuations, must be larger than 524288
        default: 1048476
      e2e_bench:
        type: boolean
        required: true
        description: Whether to run the e2e benchmark

env:
  S3_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/results
  S3_METRICS_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/metrics
  PUBLIC_S3_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/flamegraphs
  FEATURE_FLAGS: "bench-metrics,parallel,nightly-features"
  CMD_ARGS: ""
  INPUT_ARGS: ""
  CARGO_NET_GIT_FETCH_WITH_CLI: "true"

permissions:
  contents: write

jobs:
  bench-new:
    name: Run benchmark on workflow ref/branch
    runs-on:
      - runs-on
      - runner=${{ inputs.instance_type }}
      - run-id=${{ github.run_id }}
      - family=m7
      - tag=bench-${{ inputs.benchmark_name }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
    steps:
      ##########################################################################
      # Environment setup                                                      #
      ##########################################################################

      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}

      - uses: dtolnay/rust-toolchain@nightly
      - uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true

      - name: Install architecture specific tools
        run: |
          source ci/scripts/utils.sh
          install_s5cmd

      - name: Display workflow inputs
        run: echo "${{ toJSON(inputs) }}"
      - name: Feature flags
        if: contains(github.event.pull_request.labels.*.name, 'run-benchmark') || (github.event_name == 'push' && github.ref == 'refs/heads/main')
        run: |
          echo "Adding aggregation feature flag"
          echo "FEATURE_FLAGS=${FEATURE_FLAGS},aggregation" >> $GITHUB_ENV

      - name: Setup e2e (halo2 and arguments)
        run: |
          E2E_BENCH=${{ inputs.e2e_bench }}
          echo "E2E_BENCH=${E2E_BENCH}" >> $GITHUB_ENV

          if [[ "${E2E_BENCH}" == "true" ]]; then
            ROOT_ARG="--root_log_blowup ${{ inputs.root_log_blowup }}"
            INTERNAL_ARG="--internal_log_blowup ${{ inputs.internal_log_blowup }}"
            echo "INPUT_ARGS=${ROOT_ARG} ${INTERNAL_ARG} ${INPUT_ARGS}" >> $GITHUB_ENV
            bash ./extensions/native/recursion/trusted_setup_s3.sh
            export PARAMS_DIR=$(pwd)/params
          fi

      - name: Set BIN_NAME and CMD_ARGS
        run: |
          CMD_ARGS="--features ${FEATURE_FLAGS}"
          echo "CMD_ARGS=${CMD_ARGS}" >> $GITHUB_ENV
          echo "BIN_NAME=${{ inputs.benchmark_name }}" >> $GITHUB_ENV

      - name: Set working directory
        id: set-working-dir
        run: |
          WORKING_DIR=$(jq -r --arg name "${{ inputs.benchmark_name }}" '
            .benchmarks[] |
            select(.name == $name) |
            .working_directory
          ' ./ci/benchmark-config.json)
          RELATIVE_PATH=$(python3 -c "import os.path; print(os.path.relpath('.', '$WORKING_DIR'))")
          echo "working_dir=$WORKING_DIR" >> $GITHUB_OUTPUT
          echo "relative_path=$RELATIVE_PATH" >> $GITHUB_OUTPUT

      # Metric name is unique within a run (matrix)
      # When uploading to S3, use ${METRIC_NAME}-${current_sha}.[md/json]
      - name: Set metric name
        run: |
          METRIC_NAME=${{ inputs.benchmark_id || inputs.benchmark_name }}
          echo "METRIC_NAME=${METRIC_NAME}" >> $GITHUB_ENV
          METRIC_PATH=".bench_metrics/${METRIC_NAME}.json"
          echo "METRIC_PATH=${METRIC_PATH}" >> $GITHUB_ENV

      - name: Set input args
        run: |
          INSTANCE_TYPE="--instance_type ${{ inputs.instance_type }}"
          MEMORY_ALLOCATOR="--memory_allocator ${{ inputs.memory_allocator }}"
          APP_ARG="--app_log_blowup ${{ inputs.app_log_blowup }}"
          AGG_ARG="--agg_log_blowup ${{ inputs.agg_log_blowup }}"
          MAX_SEGMENT_LENGTH="--max_segment_length ${{ inputs.max_segment_length }}"
          OUTPUT_PATH="--output_path $METRIC_PATH"
          echo "INPUT_ARGS=${INSTANCE_TYPE} ${MEMORY_ALLOCATOR} ${APP_ARG} ${AGG_ARG} ${MAX_SEGMENT_LENGTH} ${OUTPUT_PATH} ${INPUT_ARGS}" >> $GITHUB_ENV

      ##########################################################################
      # Find working directory based on benchmark_name and run the benchmark   #
      ##########################################################################
      - name: Run benchmark
        working-directory: ${{ steps.set-working-dir.outputs.working_dir }}
        run: |
          python3 ${{ steps.set-working-dir.outputs.relative_path }}/ci/scripts/bench.py $BIN_NAME $CMD_ARGS $INPUT_ARGS

      ##########################################################################
      # Generate result .md files and flamegraphs, store them in S3            #
      ##########################################################################
      - name: Upload metric json and compute diff with previous to generate markdown
        run: |
          current_sha=$(git rev-parse HEAD)
          echo "Current SHA: $current_sha"
          echo "current_sha=${current_sha}" >> $GITHUB_ENV

          s5cmd cp $METRIC_PATH ${{ env.S3_METRICS_PATH }}/${METRIC_NAME}-${current_sha}.json

          source ci/scripts/utils.sh
          generate_markdown $METRIC_PATH $METRIC_NAME ${{ env.S3_METRICS_PATH }} "."

      # - name: Install inferno-flamegraph
      #   run: cargo install inferno

      # - name: Generate flamegraphs
      #   run: |
      #     if [[ -f $METRIC_PATH ]]; then
      #       python3 ci/scripts/metric_unify/flamegraph.py $METRIC_PATH
      #       s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.PUBLIC_S3_PATH }}/${current_sha}/"
      #       echo "UPLOAD_FLAMEGRAPHS=1" >> $GITHUB_ENV
      #     fi

      - name: Add benchmark metadata and upload markdown
        id: add_metadata
        run: |
          source ci/scripts/utils.sh
          add_metadata results.md ${{ inputs.max_segment_length }} ${{ inputs.instance_type }} ${{ inputs.memory_allocator }} ${{ github.repository }} ${{ github.run_id }}
          s3_md_file="${METRIC_NAME}-${current_sha}.md"
          s5cmd cp results.md "${{ env.S3_PATH }}/${s3_md_file}"

      ##########################################################################
      # Update S3 with individual results upon a push event                    #
      ##########################################################################
      - name: Update latest main result in s3
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        run: |
          s5cmd cp "${{ env.S3_PATH }}/${METRIC_NAME}-${{ env.current_sha }}.md" "${{ env.S3_PATH }}/main-${METRIC_NAME}.md"
          if [[ -f $METRIC_PATH ]]; then
            s5cmd cp $METRIC_PATH "${{ env.S3_METRICS_PATH }}/main-${METRIC_NAME}.json"
          fi

      ##########################################################################
      # Update benchmark-results with individual results                       #
      ##########################################################################
      - uses: actions/checkout@v4
        with:
          ref: benchmark-results

      - name: Set up git
        run: |
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git config --global user.name "github-actions[bot]"

      - name: Set github pages path for dispatch
        run: |
          BENCHMARK_RESULTS_PATH="benchmarks-dispatch/${{ github.head_ref || github.ref }}"
          echo "BENCHMARK_RESULTS_PATH=${BENCHMARK_RESULTS_PATH}" >> $GITHUB_ENV

      - name: Set github pages path for PR
        if: github.event_name == 'pull_request'
        run: |
          BENCHMARK_RESULTS_PATH="benchmarks-pr/${{ github.event.pull_request.number }}/individual"
          echo "BENCHMARK_RESULTS_PATH=${BENCHMARK_RESULTS_PATH}" >> $GITHUB_ENV

      - name: Set github pages path for push
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        run: |
          BENCHMARK_RESULTS_PATH="benchmarks/individual"
          echo "BENCHMARK_RESULTS_PATH=${BENCHMARK_RESULTS_PATH}" >> $GITHUB_ENV

      - name: Update PR github pages with new bench results
        if: github.event.pull_request.head.repo.fork == false # forks do not have write access
        run: |
          mkdir -p ${BENCHMARK_RESULTS_PATH}
          s3_md_file="${METRIC_NAME}-${current_sha}.md"
          s5cmd cp "${{ env.S3_PATH }}/${s3_md_file}" ${BENCHMARK_RESULTS_PATH}/${s3_md_file}
          git add ${BENCHMARK_RESULTS_PATH}/${s3_md_file}
          git commit --allow-empty -m "Update benchmark result at ${BENCHMARK_RESULTS_PATH}/${s3_md_file}"

          MAX_RETRIES=10
          RETRY_DELAY=5
          ATTEMPT=0
          SUCCESS=false

          while [ $ATTEMPT -lt $MAX_RETRIES ]; do
              echo "Attempt $((ATTEMPT + 1)) to push of $MAX_RETRIES..."
              git fetch origin benchmark-results
              git merge origin/benchmark-results --no-edit
              if git push origin benchmark-results; then
                  SUCCESS=true
                  break
              else
                  echo "Push failed. Retrying in $RETRY_DELAY seconds..."
                  sleep $RETRY_DELAY
                  ATTEMPT=$((ATTEMPT + 1))
              fi
          done
          if [ "$SUCCESS" = false ]; then
              echo "PUSH_FAILED"
              exit 1
          fi