chore: add timeouts to reports CI

.github/workflows/reports.yml

@@ -318,6 +318,18 @@ jobs:
           ./compilation_report.sh 1 ${{ matrix.project.num_runs }}
         env:
           FLAGS: ${{ matrix.project.flags }}
+
+      - name: Check compilation time limit
+        run: |
+          TIME=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/compilation_report.json)
+          TIME_LIMIT=80
+          if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "$TIME_LIMIT"; then
+            # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
+            echo "Failing due to compilation exceeding timeout..."
+            echo "Timeout: "$TIME_LIMIT"s"
+            echo "Compilation took: "$TIME"s".
+            exit 1
+          fi
 
       - name: Generate execution report
         working-directory: ./test-repo/${{ matrix.project.path }}
@@ -326,6 +338,19 @@ jobs:
           mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh
           mv /home/runner/work/noir/noir/scripts/test_programs/parse_time.sh ./parse_time.sh
           ./execution_report.sh 1 ${{ matrix.project.num_runs }}
+
+      - name: Check execution time limit
+        if: ${{ !matrix.project.cannot_execute }}
+        run: |
+          TIME=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/execution_report.json)
+          TIME_LIMIT=60
+          if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "$TIME_LIMIT"; then
+            # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
+            echo "Failing due to execution exceeding timeout..."
+            echo "Timeout: "$TIME_LIMIT"s"
+            echo "Execution took: "$TIME"s".
+            exit 1
+          fi
 
       - name: Move compilation report
         id: compilation_report
@@ -416,12 +441,37 @@ jobs:
         env:
           FLAGS: ${{ matrix.project.flags }}
 
+      - name: Check compilation memory limit
+        run: |
+          MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/compilation_memory_report.json)
+          MEMORY_LIMIT=6000
+          if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$MEMORY" "$MEMORY_LIMIT"; then
+            # Don't bump this limit without understanding why this has happened and confirming that you're not the cause.
+            echo "Failing due to compilation exceeding memory limit..."
+            echo "Limit: "$MEMORY_LIMIT"MB"
+            echo "Compilation took: "$MEMORY"MB".
+            exit 1
+          fi
+
       - name: Generate execution memory report
         working-directory: ./test-repo/${{ matrix.project.path }}
         if: ${{ !matrix.project.cannot_execute }}
         run: |
           ./memory_report.sh 1 1
 
+      - name: Check execution memory limit
+        if: ${{ !matrix.project.cannot_execute }}
+        run: |
+          MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/memory_report.json)
+          MEMORY_LIMIT=1300
+          if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$MEMORY" "$MEMORY_LIMIT"; then
+            # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
+            echo "Failing due to execution exceeding memory limit..."
+            echo "Limit: "$MEMORY_LIMIT"MB"
+            echo "Execution took: "$MEMORY"MB".
+            exit 1
+          fi
+
       - name: Move compilation report
         id: compilation_mem_report
         shell: bash
@@ -442,7 +492,6 @@ jobs:
       - name: Move execution report
         id: execution_mem_report
         if: ${{ !matrix.project.cannot_execute }}
-        shell: bash
         run: |
           PACKAGE_NAME=${{ matrix.project.path }}
           PACKAGE_NAME=$(basename $PACKAGE_NAME)
@@ -493,7 +542,6 @@ jobs:
           jq --slurp '. | flatten' ./reports/* | tee time_bench.json
 
       - name: Store benchmark result
-        continue-on-error: true
         uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
         with:
           name: "Compilation Time"
@@ -543,7 +591,6 @@ jobs:
           jq --slurp '. | flatten' ./reports/* | tee memory_bench.json
 
       - name: Store benchmark result
-        continue-on-error: true
         uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
         with:
           name: "Compilation Memory"
@@ -593,7 +640,6 @@ jobs:
           jq --slurp '. | flatten' ./reports/* | tee memory_bench.json
 
       - name: Store benchmark result
-        continue-on-error: true
         uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
         with:
           name: "Execution Memory"
@@ -644,7 +690,6 @@ jobs:
           jq --slurp '. | flatten' ./reports/* | tee time_bench.json
 
       - name: Store benchmark result
-        continue-on-error: true
         uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
         with:
           name: "Execution Time"
@@ -658,3 +703,28 @@ jobs:
           fail-on-alert: false
           alert-comment-cc-users: "@TomAFrench"
           max-items-in-chart: 50
+
+  # This is a job which depends on all test jobs and reports the overall status.
+  # This allows us to add/remove test jobs without having to update the required workflows.
+  reports-end:
+    name: End
+    runs-on: ubuntu-22.04
+    # We want this job to always run (even if the dependant jobs fail) as we want this job to fail rather than skipping.
+    if: ${{ always() }}
+    needs:
+      - upload_compilation_report
+      - upload_compilation_memory_report
+      - upload_execution_report
+      - upload_execution_memory_report
+
+    steps:
+        - name: Report overall success
+          run: |
+            if [[ $FAIL == true ]]; then
+                exit 1
+            else
+                exit 0
+            fi
+          env:
+            # We treat any skipped or failing jobs as a failure for the workflow as a whole.
+            FAIL: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }}

.github/workflows/test-js-packages.yml

@@ -527,12 +527,6 @@ jobs:
           nargo test --silence-warnings --skip-brillig-constraints-check --format json ${{ matrix.project.nargo_args }} | tee $output_file
           TIME=$(($SECONDS-$BEFORE))
 
-          if [ "TIME" -gt "${{ matrix.project.timeout }}" ]; then
-            # Library testing time has regressed past the set timeout.
-            # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
-            exit 1
-          fi
-
           NAME=${{ matrix.project.repo }}/${{ matrix.project.path }}
           # Replace any slashes with underscores
           NAME=${NAME//\//_}
@@ -547,6 +541,17 @@ jobs:
           fi
         env:
           NARGO_IGNORE_TEST_FAILURES_FROM_FOREIGN_CALLS: true
+
+      - name: Check test time limit
+        run: |
+          TIME=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/${{ steps.test_report.outputs.test_report_name }}.json)
+          if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "${{ matrix.project.timeout }}"; then
+            # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
+            echo "Failing due to test suite exceeding timeout..."
+            echo "Timeout: ${{ matrix.project.timeout }}"
+            echo "Test suite took: $TIME".
+            exit 1
+          fi
 
       - name: Compare test results
         working-directory: ./noir-repo

EXTERNAL_NOIR_LIBRARIES.yml

@@ -23,8 +23,10 @@ libraries:
     timeout: 2
   noir-bignum:
     repo: noir-lang/noir-bignum
+    timeout: 360
   noir_bigcurve:
     repo: noir-lang/noir_bigcurve
+    timeout: 290
   noir_base64:
     repo: noir-lang/noir_base64
     timeout: 2
@@ -39,7 +41,7 @@ libraries:
     timeout: 2
   noir_json_parser:
     repo: noir-lang/noir_json_parser
-    timeout: 10
+    timeout: 12
   sha256:
     repo: noir-lang/sha256
     timeout: 3
@@ -50,7 +52,7 @@ libraries:
   noir_contracts:
     repo: AztecProtocol/aztec-packages
     path: noir-projects/noir-contracts
-    timeout: 60
+    timeout: 80
   blob:
     repo: AztecProtocol/aztec-packages
     path: noir-projects/noir-protocol-circuits/crates/blob