From 8040bb158010ac630f2a8c65d68a98f13cfe2b69 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 22 Feb 2024 12:09:33 +0000 Subject: [PATCH 1/9] refactor: remove WASMTIME_ENV_HACK Closes AztecProtocol/barretenberg#837 --- barretenberg/cpp/CMakePresets.json | 16 ---------- .../cpp/scripts/_benchmark_remote_lock.sh | 1 + barretenberg/cpp/scripts/benchmark_remote.sh | 3 +- barretenberg/cpp/scripts/benchmark_wasm.sh | 9 +++--- .../cpp/scripts/benchmark_wasm_remote.sh | 15 ++++++--- .../barretenberg/env/hardware_concurrency.cpp | 12 +++---- .../cpp/src/barretenberg/srs/global_crs.cpp | 32 ------------------- 7 files changed, 22 insertions(+), 66 deletions(-) diff --git a/barretenberg/cpp/CMakePresets.json b/barretenberg/cpp/CMakePresets.json index 67b3ad028e0..002a19d456c 100644 --- a/barretenberg/cpp/CMakePresets.json +++ b/barretenberg/cpp/CMakePresets.json @@ -274,16 +274,6 @@ "MULTITHREADING": "ON" } }, - { - "name": "wasm-bench", - "displayName": "WASM benchmarking.", - "description": "WASM benchmarking.", - "inherits": "wasm-threads", - "binaryDir": "build-wasm-bench", - "environment": { - "CXXFLAGS": "-DWASMTIME_ENV_HACK" - } - }, { "name": "xray", "displayName": "Build with multi-threaded XRay Profiling", @@ -444,12 +434,6 @@ "jobs": 0, "targets": ["barretenberg.wasm"] }, - { - "name": "wasm-bench", - "configurePreset": "wasm-bench", - "inheritConfigureEnvironment": true, - "jobs": 0 - }, { "name": "xray", "configurePreset": "xray", diff --git a/barretenberg/cpp/scripts/_benchmark_remote_lock.sh b/barretenberg/cpp/scripts/_benchmark_remote_lock.sh index 551a26b9070..8ed0787051a 100644 --- a/barretenberg/cpp/scripts/_benchmark_remote_lock.sh +++ b/barretenberg/cpp/scripts/_benchmark_remote_lock.sh @@ -23,5 +23,6 @@ echo "Benchmarking lock created at ~/BENCHMARK_IN_PROGRESS." # Trap to ensure cleanup runs on ANY exit, including from a signal trap cleanup EXIT +trap cleanup INT # handle ctrl-c # don't exit, the caller script will run \ No newline at end of file diff --git a/barretenberg/cpp/scripts/benchmark_remote.sh b/barretenberg/cpp/scripts/benchmark_remote.sh index 05a61392584..b892d505ffb 100755 --- a/barretenberg/cpp/scripts/benchmark_remote.sh +++ b/barretenberg/cpp/scripts/benchmark_remote.sh @@ -11,6 +11,7 @@ BENCHMARK=${1:-goblin_bench} COMMAND=${2:-./$BENCHMARK} PRESET=${3:-clang16} BUILD_DIR=${4:-build} +HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} # Move above script dir. cd $(dirname $0)/.. @@ -24,4 +25,4 @@ source scripts/_benchmark_remote_lock.sh cd $BUILD_DIR scp $BB_SSH_KEY ./bin/$BENCHMARK $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/build ssh $BB_SSH_KEY $BB_SSH_INSTANCE \ - "cd $BB_SSH_CPP_PATH/build ; $COMMAND" + "cd $BB_SSH_CPP_PATH/build ; HARDWARE_CONCURRENCY=$HARDWARE_CONCURRENCY $COMMAND" diff --git a/barretenberg/cpp/scripts/benchmark_wasm.sh b/barretenberg/cpp/scripts/benchmark_wasm.sh index a7565485bdf..91f5d25b2c4 100755 --- a/barretenberg/cpp/scripts/benchmark_wasm.sh +++ b/barretenberg/cpp/scripts/benchmark_wasm.sh @@ -3,15 +3,16 @@ set -eu BENCHMARK=${1:-goblin_bench} COMMAND=${2:-./bin/$BENCHMARK} +HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} # Move above script dir. cd $(dirname $0)/.. # Configure and build. -cmake --preset wasm-bench -cmake --build --preset wasm-bench --target $BENCHMARK +cmake --preset wasm-threads +cmake --build --preset wasm-threads --target $BENCHMARK -cd build-wasm-bench +cd build-wasm-threads # Consistency with _wasm.sh targets / shorter $COMMAND. cp ./bin/$BENCHMARK . -wasmtime run -Wthreads=y -Sthreads=y $COMMAND \ No newline at end of file +wasmtime run --env HARDWARE_CONCURRENCY=$HARDWARE_CONCURRENCY -Wthreads=y -Sthreads=y --dir=.. $COMMAND \ No newline at end of file diff --git a/barretenberg/cpp/scripts/benchmark_wasm_remote.sh b/barretenberg/cpp/scripts/benchmark_wasm_remote.sh index f131cc63a6f..62213556579 100755 --- a/barretenberg/cpp/scripts/benchmark_wasm_remote.sh +++ b/barretenberg/cpp/scripts/benchmark_wasm_remote.sh @@ -9,17 +9,22 @@ set -eu BENCHMARK=${1:-goblin_bench} COMMAND=${2:-./$BENCHMARK} +HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} # Move above script dir. cd $(dirname $0)/.. # Configure and build. -cmake --preset wasm-bench -cmake --build --preset wasm-bench --target $BENCHMARK +cmake --preset wasm-threads +cmake --build --preset wasm-threads --target $BENCHMARK source scripts/_benchmark_remote_lock.sh -cd build-wasm-bench -scp $BB_SSH_KEY ./bin/$BENCHMARK $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/build-wasm-bench +cd build-wasm-threads +# ensure folder structure +ssh $BB_SSH_KEY $BB_SSH_INSTANCE "mkdir -p $BB_SSH_CPP_PATH/build-wasm-threads" +# copy build wasm threads +scp $BB_SSH_KEY ./bin/$BENCHMARK $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/build-wasm-threads +# run wasm benchmarking ssh $BB_SSH_KEY $BB_SSH_INSTANCE \ - "cd $BB_SSH_CPP_PATH/build-wasm-bench ; /home/ubuntu/.wasmtime/bin/wasmtime run -Wthreads=y -Sthreads=y $COMMAND" + "cd $BB_SSH_CPP_PATH/build-wasm-threads ; /home/ubuntu/.wasmtime/bin/wasmtime run --env HARDWARE_CONCURRENCY=$HARDWARE_CONCURRENCY -Wthreads=y -Sthreads=y --dir=.. $COMMAND" diff --git a/barretenberg/cpp/src/barretenberg/env/hardware_concurrency.cpp b/barretenberg/cpp/src/barretenberg/env/hardware_concurrency.cpp index 7814a02e406..9217e014eb9 100644 --- a/barretenberg/cpp/src/barretenberg/env/hardware_concurrency.cpp +++ b/barretenberg/cpp/src/barretenberg/env/hardware_concurrency.cpp @@ -1,6 +1,7 @@ #include "hardware_concurrency.hpp" #include #include +#include #include #include #include @@ -12,14 +13,9 @@ uint32_t env_hardware_concurrency() #ifndef __wasm__ try { #endif -#ifdef WASMTIME_ENV_HACK - // TODO(https://github.com/AztecProtocol/barretenberg/issues/837): Undo this hack, rely on WASI. - return 16; -#else - static auto val = std::getenv("HARDWARE_CONCURRENCY"); - static const uint32_t cores = val ? (uint32_t)std::stoul(val) : std::thread::hardware_concurrency(); - return cores; -#endif + static auto val = std::getenv("HARDWARE_CONCURRENCY"); + static const uint32_t cores = val ? (uint32_t)std::stoul(val) : std::thread::hardware_concurrency(); + return cores; #ifndef __wasm__ } catch (std::exception const&) { throw std::runtime_error("HARDWARE_CONCURRENCY invalid."); diff --git a/barretenberg/cpp/src/barretenberg/srs/global_crs.cpp b/barretenberg/cpp/src/barretenberg/srs/global_crs.cpp index fcfbac3e30c..0d09ee6c2e9 100644 --- a/barretenberg/cpp/src/barretenberg/srs/global_crs.cpp +++ b/barretenberg/cpp/src/barretenberg/srs/global_crs.cpp @@ -24,23 +24,7 @@ void init_crs_factory(std::string crs_path) if (crs_factory != nullptr) { return; } -#ifdef WASMTIME_ENV_HACK - static_cast(crs_path); - // We only need this codepath in wasmtime because the SRS cannot be loaded in our usual ways - // and we don't need a real CRS for our purposes. - // TODO(https://github.com/AztecProtocol/barretenberg/issues/837): make this a real SRS. - std::cout << "WASMTIME_ENV_HACK: started generating fake bn254 curve" << std::endl; - std::vector points; - // 2**19 points - points.reserve(1 << 19); - for (int i = 0; i < (1 << 19); i++) { - points.push_back(g1::affine_element::random_element()); - } - init_crs_factory(points, g2::affine_element{ fq::random_element(), fq::random_element() }); - std::cout << "WASMTIME_ENV_HACK: finished generating fake bn254 curve" << std::endl; -#else crs_factory = std::make_shared>(crs_path); -#endif } // Initializes the crs using the memory buffers @@ -54,23 +38,7 @@ void init_grumpkin_crs_factory(std::string crs_path) if (grumpkin_crs_factory != nullptr) { return; } -#ifdef WASMTIME_ENV_HACK - // We only need this codepath in wasmtime because the SRS cannot be loaded in our usual ways - // and we don't need a real CRS for our purposes. - // TODO(https://github.com/AztecProtocol/barretenberg/issues/837): make this a real SRS. - static_cast(crs_path); - std::cout << "WASMTIME_ENV_HACK: started generating fake grumpkin curve" << std::endl; - std::vector points; - // 2**18 points - points.reserve(1 << 18); - for (int i = 0; i < (1 << 18); i++) { - points.push_back(curve::Grumpkin::AffineElement::random_element()); - } - std::cout << "WASMTIME_ENV_HACK: finished generating fake grumpkin curve" << std::endl; - init_grumpkin_crs_factory(points); -#else grumpkin_crs_factory = std::make_shared>(crs_path); -#endif } std::shared_ptr> get_crs_factory() From be66f060aa833e6a5a601174c3579f337f206826 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 22 Feb 2024 12:58:22 +0000 Subject: [PATCH 2/9] [ci rebuild] From dabad6e5882d836c33b98dc8e6de3c0a166663bc Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 22 Feb 2024 14:54:22 +0000 Subject: [PATCH 3/9] [ci rebuild yarn-project-base] From 8762be4eaaabd195d29a0d00c8c40530f4c5118b Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 22 Feb 2024 14:54:51 +0000 Subject: [PATCH 4/9] [ci rebuild yarn-project-base] From 560b700ea2bcd5d7395077796c620e5c01a5ff60 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 22 Feb 2024 15:15:12 +0000 Subject: [PATCH 5/9] yarn project base workaround --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1cec4bf45e..9374cd6d548 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -379,15 +379,15 @@ jobs: command: build boxes-files yarn-project-base: - machine: - image: ubuntu-2204:2023.07.2 + docker: + - image: aztecprotocol/alpine-build-image resource_class: large steps: - *checkout - *setup_env - run: name: "Build" - command: build yarn-project-base | add_timestamps + command: cond_spot_run_build yarn-project-base | add_timestamps yarn-project: machine: From 4f563cf2d75d70b271537a748c7b857b02c7f91f Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 22 Feb 2024 15:18:47 +0000 Subject: [PATCH 6/9] yarn project base workaround --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9374cd6d548..02f20636e87 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -387,7 +387,7 @@ jobs: - *setup_env - run: name: "Build" - command: cond_spot_run_build yarn-project-base | add_timestamps + command: cond_spot_run_build yarn-project-base 8 | add_timestamps yarn-project: machine: From 6286c7ee39ca3095ccc3ae838bb7db53044b9d10 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 22 Feb 2024 15:21:11 +0000 Subject: [PATCH 7/9] [ci rebuild yarn-project-base] From fbada6f5d8c69abbe596987d8c4155fa1d803004 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 22 Feb 2024 15:31:53 +0000 Subject: [PATCH 8/9] fix: update --- yarn-project/end-to-end/package.json | 2 +- yarn-project/yarn.lock | 40 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/yarn-project/end-to-end/package.json b/yarn-project/end-to-end/package.json index 439ef3f006d..77bf7357f26 100644 --- a/yarn-project/end-to-end/package.json +++ b/yarn-project/end-to-end/package.json @@ -56,7 +56,7 @@ "lodash.every": "^4.6.0", "memdown": "^6.1.1", "process": "^0.11.10", - "puppeteer": "^21.3.4", + "puppeteer": "21.11.0", "resolve-typescript-plugin": "^2.0.1", "stream-browserify": "^3.0.0", "string-argv": "^0.3.2", diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 5e177a93a81..cd4ffc06583 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -414,7 +414,7 @@ __metadata: lodash.every: ^4.6.0 memdown: ^6.1.1 process: ^0.11.10 - puppeteer: ^21.3.4 + puppeteer: 21.11.0 resolve-typescript-plugin: ^2.0.1 stream-browserify: ^3.0.0 string-argv: ^0.3.2 @@ -5038,15 +5038,15 @@ __metadata: languageName: node linkType: hard -"chromium-bidi@npm:0.5.4": - version: 0.5.4 - resolution: "chromium-bidi@npm:0.5.4" +"chromium-bidi@npm:0.5.8": + version: 0.5.8 + resolution: "chromium-bidi@npm:0.5.8" dependencies: mitt: 3.0.1 - urlpattern-polyfill: 9.0.0 + urlpattern-polyfill: 10.0.0 peerDependencies: devtools-protocol: "*" - checksum: ac256ae94a9f792a220303379c81a7cff49f5aedd07dd0fcd0cd28c79b204a468ef26a9ef6fa9132d65a477ee624fff6372bebdd51b88cb4e7c7ad2275fb9b77 + checksum: 4e4bb8d3907b3a17adf15e95220fc16733e0c4b03d0eccb071010a78ca85d122a0435da95ab657ade6d15e8c9b3cd917088b6128a0aec45e4444916625a4076f languageName: node linkType: hard @@ -11290,30 +11290,30 @@ __metadata: languageName: node linkType: hard -"puppeteer-core@npm:21.9.0": - version: 21.9.0 - resolution: "puppeteer-core@npm:21.9.0" +"puppeteer-core@npm:21.11.0": + version: 21.11.0 + resolution: "puppeteer-core@npm:21.11.0" dependencies: "@puppeteer/browsers": 1.9.1 - chromium-bidi: 0.5.4 + chromium-bidi: 0.5.8 cross-fetch: 4.0.0 debug: 4.3.4 devtools-protocol: 0.0.1232444 ws: 8.16.0 - checksum: 7c92cc0bbbb33fc647ec7dde2902cca47732a5cba65bf95bd46f352c9275e30097ae26a42714d7b6407c83f1940bad82f42d8640e3051efb309fe47b8385ceea + checksum: 953018ca679542d747e5974fd732349d73f4c685c0b49dd6027d493ad95d64a23b2d6e3e72f0a0a12c7335d3e08a0b3e6bc9ac3adfb3fb0316fb1ca62f46f129 languageName: node linkType: hard -"puppeteer@npm:^21.3.4": - version: 21.9.0 - resolution: "puppeteer@npm:21.9.0" +"puppeteer@npm:21.11.0": + version: 21.11.0 + resolution: "puppeteer@npm:21.11.0" dependencies: "@puppeteer/browsers": 1.9.1 cosmiconfig: 9.0.0 - puppeteer-core: 21.9.0 + puppeteer-core: 21.11.0 bin: puppeteer: lib/esm/puppeteer/node/cli.js - checksum: 0cb28d29a183af04869022ff30632116be65f1f8ec4ba729cc36a22332aefee697909adc7c4a15ceae53fdf657c9450aa9b8d8eacab05535adf4972ef2e50038 + checksum: 0f2e6f6d9269e511022ec8e8478f42bf5f3dbddbdac7b50d31c90504993eaef436e8729a5b3f105576ec2031519de723d4a9d0ad797821d0f16540073fabc37a languageName: node linkType: hard @@ -13261,10 +13261,10 @@ __metadata: languageName: node linkType: hard -"urlpattern-polyfill@npm:9.0.0": - version: 9.0.0 - resolution: "urlpattern-polyfill@npm:9.0.0" - checksum: d3658b78a10eaee514c464f5a4336c408c70cf01e9b915cb1df5892b3c49003d1ed4042dc72d1b18493b8b847883e84fbf2bf358abb5dff84b2725d5e8463bcb +"urlpattern-polyfill@npm:10.0.0": + version: 10.0.0 + resolution: "urlpattern-polyfill@npm:10.0.0" + checksum: 61d890f151ea4ecf34a3dcab32c65ad1f3cda857c9d154af198260c6e5b2ad96d024593409baaa6d4428dd1ab206c14799bf37fe011117ac93a6a44913ac5aa4 languageName: node linkType: hard From 74d49b955070cd9e21d05d6a46c9d21f866665d0 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 10:27:12 +0000 Subject: [PATCH 9/9] revert accident --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 02f20636e87..a1cec4bf45e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -379,15 +379,15 @@ jobs: command: build boxes-files yarn-project-base: - docker: - - image: aztecprotocol/alpine-build-image + machine: + image: ubuntu-2204:2023.07.2 resource_class: large steps: - *checkout - *setup_env - run: name: "Build" - command: cond_spot_run_build yarn-project-base 8 | add_timestamps + command: build yarn-project-base | add_timestamps yarn-project: machine: