diff --git a/.github/scripts/merge-bench-reports.sh b/.github/scripts/merge-bench-reports.sh new file mode 100755 index 00000000000..23a62874148 --- /dev/null +++ b/.github/scripts/merge-bench-reports.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -eu + +echo "Merging reports" + +REPORT_NAME=$1 +NAME_PLURAL=""$REPORT_NAME"s" + +combined_reports="[]" + +# Iterate over each report and merge them +for report in ./reports/*; do + # The report is saved under ./$REPORT_NAME_{ matrix_report }/$REPORT_NAME_{ matrix_report }.json + FILE_PATH=$(echo $(ls $report)) + + # Extract the $NAME_PLURAL array from each report and merge it + combined_reports=$(jq '[."'"$NAME_PLURAL"'"[]] + '"$combined_reports" <<< "$(cat "$report/$FILE_PATH")") +done + +combined_reports=$(jq '[."'$NAME_PLURAL'"[]] + '"$combined_reports" <<< "$(cat ./$REPORT_NAME.json)") + +# Wrap the merged memory reports into a new object as to keep the $NAME_PLURAL key +final_report="{\"$NAME_PLURAL\": $combined_reports}" + +echo "$final_report" > $REPORT_NAME.json + +cat $REPORT_NAME.json \ No newline at end of file diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index c84b32dbd83..314a424327b 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -217,21 +217,13 @@ jobs: ./memory_report.sh mv memory_report.json ../memory_report.json - - name: Parse memory report - id: memory_report - uses: noir-lang/noir-bench-report@0d7464a8c39170523932d7846b6e6b458a294aea - with: - report: memory_report.json - header: | - # Memory Report - memory_report: true - - - name: Add memory report to sticky comment - if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' - uses: marocchino/sticky-pull-request-comment@v2 + - name: Upload memory report + uses: actions/upload-artifact@v4 with: - header: memory - message: ${{ steps.memory_report.outputs.markdown }} + name: in_progress_memory_report + path: memory_report.json + retention-days: 3 + overwrite: true generate_compilation_report: name: Compilation time @@ -283,10 +275,11 @@ jobs: - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts } - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root } - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail } - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset } # TODO: Bring these back once they no longer time out - #- project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private } - #- project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public } + # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private } + # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public } name: External repo compilation report - ${{ matrix.project.repo }}/${{ matrix.project.path }} steps: @@ -367,26 +360,8 @@ jobs: - name: Merge compilation reports using jq run: | - echo "Merging reports" - - combined_reports="[]" - - # Iterate over each report and merge them - for report in ./reports/*; do - # The report is saved under ./compilation_report_{ matrix_report }/compilation_report_{ matrix_report }.json - REPORT_NAME=$(echo $(ls $report)) - # Extract the 'compilation_reports' array from each report and merge it - combined_reports=$(jq '[.compilation_reports[]] + '"$combined_reports" <<< "$(cat "$report/$REPORT_NAME")") - done - - combined_reports=$(jq '[.compilation_reports[]] + '"$combined_reports" <<< "$(cat ./compilation_report.json)") - - # Wrap the merged compilation reports into a new object as to keep the 'compilation_reports' key - final_report="{\"compilation_reports\": $combined_reports}" - - echo "$final_report" > compilation_report.json - - cat compilation_report.json + mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh + ./merge-bench-reports.sh compilation_report - name: Parse compilation report id: compilation_report @@ -403,3 +378,116 @@ jobs: with: header: compilation message: ${{ steps.compilation_report.outputs.markdown }} + + external_repo_memory_report: + needs: [build-nargo] + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + # TODO: Bring this report back under a flag. The `noir-contracts` report takes just under 30 minutes. + # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail } + + name: External repo memory report - ${{ matrix.project.repo }}/${{ matrix.project.path }} + steps: + - name: Download nargo binary + uses: actions/download-artifact@v4 + with: + name: nargo + path: ./nargo + + - name: Set nargo on PATH + run: | + nargo_binary="${{ github.workspace }}/nargo/nargo" + chmod +x $nargo_binary + echo "$(dirname $nargo_binary)" >> $GITHUB_PATH + export PATH="$PATH:$(dirname $nargo_binary)" + nargo -V + + - uses: actions/checkout@v4 + with: + path: scripts + sparse-checkout: | + test_programs/memory_report.sh + sparse-checkout-cone-mode: false + + - name: Checkout + uses: actions/checkout@v4 + with: + repository: ${{ matrix.project.repo }} + path: test-repo + ref: ${{ matrix.project.ref }} + + - name: Generate compilation report + working-directory: ./test-repo/${{ matrix.project.path }} + run: | + mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh + chmod +x ./memory_report.sh + ./memory_report.sh 1 + + - name: Move compilation report + id: report + shell: bash + run: | + PACKAGE_NAME=${{ matrix.project.path }} + PACKAGE_NAME=$(basename $PACKAGE_NAME) + mv ./test-repo/${{ matrix.project.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json + echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT + + - name: Upload memory report + uses: actions/upload-artifact@v4 + with: + name: memory_report_${{ steps.report.outputs.memory_report_name }} + path: memory_report_${{ steps.report.outputs.memory_report_name }}.json + retention-days: 3 + overwrite: true + + upload_memory_report: + name: Upload memory report + needs: [generate_memory_report, external_repo_memory_report] + # We want this job to run even if one variation of the matrix in `external_repo_memory_report` fails + if: always() + runs-on: ubuntu-latest + permissions: + pull-requests: write + + steps: + - uses: actions/checkout@v4 + + - name: Download initial memory report + uses: actions/download-artifact@v4 + with: + name: in_progress_memory_report + + - name: Download matrix memory reports + uses: actions/download-artifact@v4 + with: + pattern: memory_report_* + path: ./reports + + - name: Merge memory reports using jq + run: | + mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh + ./merge-bench-reports.sh memory_report + + - name: Parse memory report + id: memory_report + uses: noir-lang/noir-bench-report@0d7464a8c39170523932d7846b6e6b458a294aea + with: + report: memory_report.json + header: | + # Memory Report + memory_report: true + + - name: Add memory report to sticky comment + if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: memory + message: ${{ steps.memory_report.outputs.markdown }} diff --git a/test_programs/memory_report.sh b/test_programs/memory_report.sh index 1b8274b76cc..4d03726d374 100755 --- a/test_programs/memory_report.sh +++ b/test_programs/memory_report.sh @@ -5,22 +5,24 @@ sudo apt-get install heaptrack NARGO="nargo" - # Tests to be profiled for memory report tests_to_profile=("keccak256" "workspace" "regression_4709" "ram_blowup_regression") current_dir=$(pwd) -execution_success_path="$current_dir/execution_success" -test_dirs=$(ls $execution_success_path) +base_path="$current_dir/execution_success" + +# If there is an argument that means we want to generate a report for only the current directory +if [ "$#" -ne 0 ]; then + base_path="$current_dir" + tests_to_profile=(".") +fi FIRST="1" echo "{\"memory_reports\": [ " > memory_report.json - for test_name in ${tests_to_profile[@]}; do - full_path=$execution_success_path"/"$test_name - cd $full_path + cd $base_path/$test_name if [ $FIRST = "1" ] then @@ -28,6 +30,11 @@ for test_name in ${tests_to_profile[@]}; do else echo " ," >> $current_dir"/memory_report.json" fi + + if [ "$#" -ne 0 ]; then + test_name=$(basename $current_dir) + fi + heaptrack --output $current_dir/$test_name"_heap" $NARGO compile --force if test -f $current_dir/$test_name"_heap.gz"; then