From 4d107e15244a7dfda4fcd44ed465d6ae816790e2 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 30 Oct 2024 01:12:48 +0000 Subject: [PATCH 01/11] run masterly sanitizers --- .github/workflows/bb-msan.yml | 25 ------- .github/workflows/bb-sanitizers.yml | 71 ++++++++++++++++++++ .github/workflows/publish-aztec-packages.yml | 1 - barretenberg/cpp/Earthfile | 10 ++- 4 files changed, 80 insertions(+), 27 deletions(-) delete mode 100644 .github/workflows/bb-msan.yml create mode 100644 .github/workflows/bb-sanitizers.yml diff --git a/.github/workflows/bb-msan.yml b/.github/workflows/bb-msan.yml deleted file mode 100644 index fe30b2193a4..00000000000 --- a/.github/workflows/bb-msan.yml +++ /dev/null @@ -1,25 +0,0 @@ -# Checks bb prover with memory sanitizer -name: BB MSAN -on: - pull_request: - types: [opened, synchronize, reopened, labeled] - paths: - - 'barretenberg/**' - workflow_dispatch: - inputs: {} -# unlike most jobs, we have no concurrency limits - note that we should have spare capacity (of total 1000 concurrency) -# to run a good number of these, but we should still be mindful that we don't queue, say, dozens of these. -jobs: - bb-msan-check: - if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bb-msan-check') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - uses: earthly/actions-setup@v1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: "Barretenberg Prover MSAN Check" - timeout-minutes: 720 # 12 hour timeout. Yes, it's slow, and we just want to use the github runner. - run: earthly ./barretenberg/cpp/+preset-msan-check diff --git a/.github/workflows/bb-sanitizers.yml b/.github/workflows/bb-sanitizers.yml new file mode 100644 index 00000000000..b1f11783e3d --- /dev/null +++ b/.github/workflows/bb-sanitizers.yml @@ -0,0 +1,71 @@ +# Checks bb prover with memory sanitizer +name: BB MSAN +on: + push: + branches: + - master + - "*/*bb-sanitizers*" + pull_request: + types: [opened, synchronize, reopened, labeled] + paths: + - 'barretenberg/**' + workflow_dispatch: + inputs: {} +# unlike most jobs, we have no concurrency limits - note that we should have spare capacity (of total 1000 concurrency) +# to run a good number of these, but we should still be mindful that we don't queue, say, dozens of these. +jobs: + bb-msan-check: + if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bb-msan-check') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: earthly/actions-setup@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: "Barretenberg Prover MSAN Check" + timeout-minutes: 360 # 6 hour timeout. Yes, it's slow, and we just want to use the 4 core github runner (free). + run: earthly --no-cache ./barretenberg/cpp/+preset-msan-check + + bb-asan-check: + if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bb-asan-check') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: earthly/actions-setup@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: "Barretenberg Prover ASAN Check" + timeout-minutes: 120 # 2 hour timeout just in case. Yes, it's slow, and we just want to use the 4 core github runner (free). + run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=asan + + bb-tsan-check: + if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bb-tsan-check') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: earthly/actions-setup@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: "Barretenberg Prover TSAN Check" + timeout-minutes: 120 # 2 hour timeout just in case. Yes, it's slow, and we just want to use the 4 core github runner (free). + run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=tsan + + bb-ubsan-check: + if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bb-tsan-check') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: earthly/actions-setup@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: "Barretenberg Prover TSAN Check" + timeout-minutes: 120 # 2 hour timeout just in case. Yes, it's slow, and we just want to use the 4 core github runner (free). + run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=ubsan \ No newline at end of file diff --git a/.github/workflows/publish-aztec-packages.yml b/.github/workflows/publish-aztec-packages.yml index 9bc5f3e88fd..1fb170f4f99 100644 --- a/.github/workflows/publish-aztec-packages.yml +++ b/.github/workflows/publish-aztec-packages.yml @@ -5,7 +5,6 @@ on: branches: - master - "*/release-master*" - - ludamad-patch-2 workflow_dispatch: inputs: tag: diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index 72b3fb7f218..b98822c726b 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -107,7 +107,15 @@ preset-msan-check: # install SRS needed for proving COPY --dir ./srs_db/+build/. srs_db RUN echo "Warning: If ./bin/client_ivc_tests is not found, there may be build failures above." - RUN cd build && ./bin/client_ivc_tests --gtest_also_run_disabled_tests + RUN cd build && ./bin/client_ivc_tests + +preset-check: + ARG preset + FROM +source + RUN cmake --preset $preset -Bbuild && cmake --build build --target client_ivc_tests + # install SRS needed for proving + COPY --dir ./srs_db/+build/. srs_db + RUN cd build && ./bin/client_ivc_tests preset-wasm: ARG TARGETARCH From 2404dc3812aca6d45168af1d8f28a249441d7ab0 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 30 Oct 2024 01:13:23 +0000 Subject: [PATCH 02/11] fix --- .github/workflows/bb-sanitizers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bb-sanitizers.yml b/.github/workflows/bb-sanitizers.yml index b1f11783e3d..270a0199be8 100644 --- a/.github/workflows/bb-sanitizers.yml +++ b/.github/workflows/bb-sanitizers.yml @@ -57,7 +57,7 @@ jobs: run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=tsan bb-ubsan-check: - if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bb-tsan-check') + if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bb-ubsan-check') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From b1c2490ac19f7872c53dce3c417d83bc9edb62cf Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 29 Oct 2024 21:17:06 -0400 Subject: [PATCH 03/11] Update bb-sanitizers.yml --- .github/workflows/bb-sanitizers.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bb-sanitizers.yml b/.github/workflows/bb-sanitizers.yml index 270a0199be8..e9c97955ec9 100644 --- a/.github/workflows/bb-sanitizers.yml +++ b/.github/workflows/bb-sanitizers.yml @@ -4,7 +4,7 @@ on: push: branches: - master - - "*/*bb-sanitizers*" + - "*/bb-sanitizers*" pull_request: types: [opened, synchronize, reopened, labeled] paths: @@ -68,4 +68,4 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: "Barretenberg Prover TSAN Check" timeout-minutes: 120 # 2 hour timeout just in case. Yes, it's slow, and we just want to use the 4 core github runner (free). - run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=ubsan \ No newline at end of file + run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=ubsan From 448c8672ee01016b0a1e239d33139c23f14fc01f Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 29 Oct 2024 21:18:22 -0400 Subject: [PATCH 04/11] Update bb-sanitizers.yml --- .github/workflows/bb-sanitizers.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bb-sanitizers.yml b/.github/workflows/bb-sanitizers.yml index e9c97955ec9..dd7d1afde2a 100644 --- a/.github/workflows/bb-sanitizers.yml +++ b/.github/workflows/bb-sanitizers.yml @@ -15,7 +15,7 @@ on: # to run a good number of these, but we should still be mindful that we don't queue, say, dozens of these. jobs: bb-msan-check: - if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bb-msan-check') + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-msan-check') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -29,7 +29,7 @@ jobs: run: earthly --no-cache ./barretenberg/cpp/+preset-msan-check bb-asan-check: - if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bb-asan-check') + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-asan-check') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -43,7 +43,7 @@ jobs: run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=asan bb-tsan-check: - if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bb-tsan-check') + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-tsan-check') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -57,7 +57,7 @@ jobs: run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=tsan bb-ubsan-check: - if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'bb-ubsan-check') + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-ubsan-check') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 2f49fc1f836b871f2a5b9dbadf7a39e28af05b86 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 29 Oct 2024 21:19:28 -0400 Subject: [PATCH 05/11] Update bb-sanitizers.yml --- .github/workflows/bb-sanitizers.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bb-sanitizers.yml b/.github/workflows/bb-sanitizers.yml index dd7d1afde2a..94111a9c13f 100644 --- a/.github/workflows/bb-sanitizers.yml +++ b/.github/workflows/bb-sanitizers.yml @@ -1,4 +1,6 @@ -# Checks bb prover with memory sanitizer +# Checks bb prover with sanitizers +# Unlike most jobs uses free 4 core github runners of which we have lots of capacity (of total 1000 concurrency) +# to run a good number of these, but we should still be mindful that we don't queue, say, dozens of these per person. name: BB MSAN on: push: @@ -11,8 +13,11 @@ on: - 'barretenberg/**' workflow_dispatch: inputs: {} -# unlike most jobs, we have no concurrency limits - note that we should have spare capacity (of total 1000 concurrency) -# to run a good number of these, but we should still be mindful that we don't queue, say, dozens of these. + +concurrency: + # force parallelism in master + group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} + cancel-in-progress: true jobs: bb-msan-check: if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-msan-check') From 5c15894b07f1681cc3f6889cc31fbd1ec0cd012d Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 29 Oct 2024 21:19:42 -0400 Subject: [PATCH 06/11] Update bb-sanitizers.yml --- .github/workflows/bb-sanitizers.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/bb-sanitizers.yml b/.github/workflows/bb-sanitizers.yml index 94111a9c13f..6ba2028b906 100644 --- a/.github/workflows/bb-sanitizers.yml +++ b/.github/workflows/bb-sanitizers.yml @@ -1,6 +1,5 @@ # Checks bb prover with sanitizers -# Unlike most jobs uses free 4 core github runners of which we have lots of capacity (of total 1000 concurrency) -# to run a good number of these, but we should still be mindful that we don't queue, say, dozens of these per person. +# Unlike most jobs uses free 4 core github runners of which we have lots of capacity (of total 1000 concurrency). name: BB MSAN on: push: From 38b83ec76ca520d889bb284a4e2c82da124d9bc1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 29 Oct 2024 23:31:06 -0400 Subject: [PATCH 07/11] Update bb-sanitizers.yml --- .github/workflows/bb-sanitizers.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bb-sanitizers.yml b/.github/workflows/bb-sanitizers.yml index 6ba2028b906..3250173268d 100644 --- a/.github/workflows/bb-sanitizers.yml +++ b/.github/workflows/bb-sanitizers.yml @@ -29,7 +29,7 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - name: "Barretenberg Prover MSAN Check" - timeout-minutes: 360 # 6 hour timeout. Yes, it's slow, and we just want to use the 4 core github runner (free). + run: earthly --no-cache ./barretenberg/cpp/+preset-msan-check bb-asan-check: @@ -43,7 +43,7 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - name: "Barretenberg Prover ASAN Check" - timeout-minutes: 120 # 2 hour timeout just in case. Yes, it's slow, and we just want to use the 4 core github runner (free). + timeout-minutes: 360 # 6 hour timeout (proving) run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=asan bb-tsan-check: @@ -57,7 +57,7 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - name: "Barretenberg Prover TSAN Check" - timeout-minutes: 120 # 2 hour timeout just in case. Yes, it's slow, and we just want to use the 4 core github runner (free). + timeout-minutes: 360 # 6 hour timeout (proving) run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=tsan bb-ubsan-check: @@ -71,5 +71,5 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - name: "Barretenberg Prover TSAN Check" - timeout-minutes: 120 # 2 hour timeout just in case. Yes, it's slow, and we just want to use the 4 core github runner (free). + timeout-minutes: 360 # 6 hour timeout (proving) run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=ubsan From 69cec2a45b24afc559c4aabebdc5be50fe53109d Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 30 Oct 2024 09:25:01 -0400 Subject: [PATCH 08/11] Update bb-sanitizers.yml --- .github/workflows/bb-sanitizers.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bb-sanitizers.yml b/.github/workflows/bb-sanitizers.yml index 3250173268d..0a83c7e6541 100644 --- a/.github/workflows/bb-sanitizers.yml +++ b/.github/workflows/bb-sanitizers.yml @@ -43,7 +43,7 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - name: "Barretenberg Prover ASAN Check" - timeout-minutes: 360 # 6 hour timeout (proving) + timeout-minutes: 720 # 12 hour timeout (proving) run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=asan bb-tsan-check: @@ -57,7 +57,7 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - name: "Barretenberg Prover TSAN Check" - timeout-minutes: 360 # 6 hour timeout (proving) + timeout-minutes: 720 # 12 hour timeout (proving) run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=tsan bb-ubsan-check: @@ -71,5 +71,5 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} - name: "Barretenberg Prover TSAN Check" - timeout-minutes: 360 # 6 hour timeout (proving) + timeout-minutes: 720 # 12 hour timeout (proving) run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=ubsan From 43588eb332847bf35d2ef7d127d838206939ddf7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 30 Oct 2024 19:04:53 -0400 Subject: [PATCH 09/11] Update bb-sanitizers.yml --- .github/workflows/bb-sanitizers.yml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bb-sanitizers.yml b/.github/workflows/bb-sanitizers.yml index 0a83c7e6541..06aee328c1a 100644 --- a/.github/workflows/bb-sanitizers.yml +++ b/.github/workflows/bb-sanitizers.yml @@ -1,4 +1,4 @@ -# Checks bb prover with sanitizers +# Checks bb (barretenberg prover library) prover with sanitizers # Unlike most jobs uses free 4 core github runners of which we have lots of capacity (of total 1000 concurrency). name: BB MSAN on: @@ -18,9 +18,27 @@ concurrency: group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} cancel-in-progress: true jobs: + # acts as prover performance baseline + bb-baseline: + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-msan-check') + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: earthly/actions-setup@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: "Barretenberg Prover MSAN Check" + + run: earthly --no-cache ./barretenberg/cpp/+preset-msan-check + + # memory sanitzer for prover bb-msan-check: if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-msan-check') runs-on: ubuntu-latest + continue-on-error: true steps: - uses: actions/checkout@v4 with: @@ -32,9 +50,11 @@ jobs: run: earthly --no-cache ./barretenberg/cpp/+preset-msan-check + # address sanitzer for prover bb-asan-check: if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-asan-check') runs-on: ubuntu-latest + continue-on-error: true steps: - uses: actions/checkout@v4 with: @@ -46,9 +66,11 @@ jobs: timeout-minutes: 720 # 12 hour timeout (proving) run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=asan + # address sanitzer for prover bb-tsan-check: if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-tsan-check') runs-on: ubuntu-latest + continue-on-error: true steps: - uses: actions/checkout@v4 with: @@ -60,9 +82,11 @@ jobs: timeout-minutes: 720 # 12 hour timeout (proving) run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=tsan + # undefined behavior sanitzer for prover bb-ubsan-check: if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'bb-ubsan-check') runs-on: ubuntu-latest + continue-on-error: true steps: - uses: actions/checkout@v4 with: From 34afe1a74251d03194003890510fd20bdfebf9f7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 30 Oct 2024 19:17:50 -0400 Subject: [PATCH 10/11] Update Earthfile --- barretenberg/cpp/Earthfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index b98822c726b..390a7af6544 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -107,7 +107,7 @@ preset-msan-check: # install SRS needed for proving COPY --dir ./srs_db/+build/. srs_db RUN echo "Warning: If ./bin/client_ivc_tests is not found, there may be build failures above." - RUN cd build && ./bin/client_ivc_tests + RUN cd build && ./bin/client_ivc_tests --gtest_filter="*BasicStructured" preset-check: ARG preset @@ -115,7 +115,7 @@ preset-check: RUN cmake --preset $preset -Bbuild && cmake --build build --target client_ivc_tests # install SRS needed for proving COPY --dir ./srs_db/+build/. srs_db - RUN cd build && ./bin/client_ivc_tests + RUN cd build && ./bin/client_ivc_tests --gtest_filter="*BasicStructured" preset-wasm: ARG TARGETARCH From b78ed6b7a2f9898abecde83c191f479e755658d2 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 30 Oct 2024 19:23:17 -0400 Subject: [PATCH 11/11] Update bb-sanitizers.yml --- .github/workflows/bb-sanitizers.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bb-sanitizers.yml b/.github/workflows/bb-sanitizers.yml index 06aee328c1a..84b2c61d767 100644 --- a/.github/workflows/bb-sanitizers.yml +++ b/.github/workflows/bb-sanitizers.yml @@ -30,9 +30,8 @@ jobs: - uses: earthly/actions-setup@v1 with: github-token: ${{ secrets.GITHUB_TOKEN }} - - name: "Barretenberg Prover MSAN Check" - - run: earthly --no-cache ./barretenberg/cpp/+preset-msan-check + - name: "Barretenberg Baseline Performance Check" + run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=clang16 # memory sanitzer for prover bb-msan-check: