From 5260f1e087d085ff42dc72801274442932af9778 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 20 Dec 2024 17:34:26 -0300 Subject: [PATCH] chore: Configurable parallelism in bootstrap (#10909) Bootstrap scripts defined RAYON_NUM_THREADS and HARDWARE_CONCURRENCY to 16, and also ran parallel with 16 processes. This means that a bootstrap run could take up to 256 cores. This PR loads the values for those vars from env if set, so they can be overridden, but defaults to the original values that work on CI/mainframe. Also honors the env vars in test scripts. --- noir-projects/noir-contracts/bootstrap.sh | 11 ++++++----- noir-projects/noir-protocol-circuits/bootstrap.sh | 7 ++++--- yarn-project/accounts/package.json | 2 +- yarn-project/archiver/package.json | 2 +- yarn-project/aztec-faucet/package.json | 2 +- yarn-project/aztec-node/package.json | 2 +- yarn-project/aztec.js/package.json | 2 +- yarn-project/aztec/package.json | 2 +- yarn-project/bb-prover/package.json | 2 +- yarn-project/bot/package.json | 2 +- yarn-project/builder/package.json | 2 +- yarn-project/circuit-types/package.json | 2 +- yarn-project/circuits.js/package.json | 2 +- yarn-project/cli-wallet/package.json | 2 +- yarn-project/cli/package.json | 2 +- yarn-project/entrypoints/package.json | 2 +- yarn-project/epoch-cache/package.json | 2 +- yarn-project/ethereum/package.json | 2 +- yarn-project/foundation/package.json | 2 +- yarn-project/key-store/package.json | 2 +- yarn-project/merkle-tree/package.json | 2 +- yarn-project/noir-contracts.js/package.json | 2 +- .../noir-protocol-circuits-types/package.json | 2 +- yarn-project/p2p-bootstrap/package.json | 2 +- yarn-project/p2p/package.json | 2 +- yarn-project/package.common.json | 2 +- yarn-project/proof-verifier/package.json | 2 +- yarn-project/protocol-contracts/package.json | 2 +- yarn-project/prover-node/package.json | 2 +- yarn-project/pxe/package.json | 2 +- yarn-project/scripts/package.json | 2 +- yarn-project/sequencer-client/package.json | 2 +- yarn-project/simulator/package.json | 2 +- yarn-project/telemetry-client/package.json | 2 +- yarn-project/txe/package.json | 2 +- yarn-project/types/package.json | 2 +- yarn-project/validator-client/package.json | 2 +- yarn-project/world-state/package.json | 2 +- 38 files changed, 46 insertions(+), 44 deletions(-) diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index aa43652e2d7..a441ec1c978 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -3,7 +3,7 @@ # - First of all, I'm sorry. It's a beautiful script but it's no fun to debug. I got carried away. # - You can enable BUILD_SYSTEM_DEBUG=1 but the output is quite verbose that it's not much use by default. # - You can call ./bootstrap.sh build to compile and process a single contract. -# - You can disable further parallelism by putting -j1 on the parallel calls. +# - You can disable further parallelism by setting PARALLELISM=1. # - The exported functions called by parallel must enable their own flags at the start e.g. set -euo pipefail # - The exported functions are using stdin/stdout, so be very careful about what's printed where. # - The exported functions need to have external variables they require, to have been exported first. @@ -18,8 +18,9 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap cmd=${1:-} -export RAYON_NUM_THREADS=16 -export HARDWARE_CONCURRENCY=16 +export RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-16} +export HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} +export PARALLELISM=${PARALLELISM:-16} export PLATFORM_TAG=any export BB=${BB:-../../barretenberg/cpp/build/bin/bb} @@ -116,7 +117,7 @@ function compile { # .[1] is the updated functions on stdin (-) # * merges their fields. jq -c '.functions[]' $json_path | \ - parallel -j16 --keep-order -N1 --block 8M --pipe --halt now,fail=1 process_function | \ + parallel -j$PARALLELISM --keep-order -N1 --block 8M --pipe --halt now,fail=1 process_function | \ jq -s '{functions: .}' | jq -s '.[0] * {functions: .[1].functions}' $json_path - > $tmp_dir/$filename mv $tmp_dir/$filename $json_path } @@ -131,7 +132,7 @@ function build { set +e echo_stderr "Compiling contracts (bb-hash: $BB_HASH)..." grep -oP '(?<=contracts/)[^"]+' Nargo.toml | \ - parallel -j16 --joblog joblog.txt -v --line-buffer --tag --halt now,fail=1 compile {} + parallel -j$PARALLELISM --joblog joblog.txt -v --line-buffer --tag --halt now,fail=1 compile {} code=$? cat joblog.txt return $code diff --git a/noir-projects/noir-protocol-circuits/bootstrap.sh b/noir-projects/noir-protocol-circuits/bootstrap.sh index 1abd3b83255..bbdba711507 100755 --- a/noir-projects/noir-protocol-circuits/bootstrap.sh +++ b/noir-projects/noir-protocol-circuits/bootstrap.sh @@ -4,8 +4,9 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap CMD=${1:-} -export RAYON_NUM_THREADS=16 -export HARDWARE_CONCURRENCY=16 +export RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-16} +export HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} +export PARALLELISM=${PARALLELISM:-16} export PLATFORM_TAG=any export BB=${BB:-../../barretenberg/cpp/build/bin/bb} @@ -107,7 +108,7 @@ function build { echo "$(basename $dir)" fi done | \ - parallel -j16 --joblog joblog.txt -v --line-buffer --tag --halt now,fail=1 compile {} + parallel -j$PARALLELISM --joblog joblog.txt -v --line-buffer --tag --halt now,fail=1 compile {} code=$? cat joblog.txt return $code diff --git a/yarn-project/accounts/package.json b/yarn-project/accounts/package.json index 2ee45c5862f..83ff41b5309 100644 --- a/yarn-project/accounts/package.json +++ b/yarn-project/accounts/package.json @@ -33,7 +33,7 @@ "clean": "rm -rf ./dest .tsbuildinfo ./artifacts", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json", diff --git a/yarn-project/archiver/package.json b/yarn-project/archiver/package.json index 8724a38da26..7e9a218284f 100644 --- a/yarn-project/archiver/package.json +++ b/yarn-project/archiver/package.json @@ -22,7 +22,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8", + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}", "test:integration": "concurrently -k -s first -c reset,dim -n test,anvil \"yarn test:integration:run\" \"anvil\"", "test:integration:run": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --no-cache --config jest.integration.config.json" }, diff --git a/yarn-project/aztec-faucet/package.json b/yarn-project/aztec-faucet/package.json index 63e5b0d76f3..b0be9805a99 100644 --- a/yarn-project/aztec-faucet/package.json +++ b/yarn-project/aztec-faucet/package.json @@ -22,7 +22,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/aztec-node/package.json b/yarn-project/aztec-node/package.json index b2bc307a923..6f7ec39ff16 100644 --- a/yarn-project/aztec-node/package.json +++ b/yarn-project/aztec-node/package.json @@ -23,7 +23,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/aztec.js/package.json b/yarn-project/aztec.js/package.json index 3a3d36968cf..a18a0400266 100644 --- a/yarn-project/aztec.js/package.json +++ b/yarn-project/aztec.js/package.json @@ -38,7 +38,7 @@ "clean": "rm -rf ./dest .tsbuildinfo ./src/account_contract/artifacts", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json", diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index 619a268c4ab..171642eddda 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -21,7 +21,7 @@ "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "build:dev": "tsc -b --watch", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8", + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}", "run:example:token": "LOG_LEVEL='verbose' node ./dest/examples/token.js" }, "inherits": [ diff --git a/yarn-project/bb-prover/package.json b/yarn-project/bb-prover/package.json index 1004dc1fbc9..d8ead63d687 100644 --- a/yarn-project/bb-prover/package.json +++ b/yarn-project/bb-prover/package.json @@ -30,7 +30,7 @@ "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "bb": "node --no-warnings ./dest/bb/index.js", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "jest": { "moduleNameMapper": { diff --git a/yarn-project/bot/package.json b/yarn-project/bot/package.json index e8cb69769ff..2bffdb95764 100644 --- a/yarn-project/bot/package.json +++ b/yarn-project/bot/package.json @@ -16,7 +16,7 @@ "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "bb": "node --no-warnings ./dest/bb/index.js", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "jest": { "moduleNameMapper": { diff --git a/yarn-project/builder/package.json b/yarn-project/builder/package.json index 9eb7841637d..4d646502b29 100644 --- a/yarn-project/builder/package.json +++ b/yarn-project/builder/package.json @@ -23,7 +23,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/circuit-types/package.json b/yarn-project/circuit-types/package.json index c30dbfe249e..7f6445ad14f 100644 --- a/yarn-project/circuit-types/package.json +++ b/yarn-project/circuit-types/package.json @@ -25,7 +25,7 @@ "clean": "rm -rf ./dest .tsbuildinfo ./src/test/artifacts", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8", + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}", "generate": "./scripts/copy-test-artifacts.sh && run -T prettier -w ./src/test/artifacts --loglevel warn" }, "inherits": [ diff --git a/yarn-project/circuits.js/package.json b/yarn-project/circuits.js/package.json index 3f556edd8e3..535b9124aaf 100644 --- a/yarn-project/circuits.js/package.json +++ b/yarn-project/circuits.js/package.json @@ -38,7 +38,7 @@ "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "remake-constants": "node --loader ts-node/esm src/scripts/constants.in.ts && prettier -w src/constants.gen.ts && cd ../../l1-contracts && forge fmt", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "dependencies": { "@aztec/bb.js": "portal:../../barretenberg/ts", diff --git a/yarn-project/cli-wallet/package.json b/yarn-project/cli-wallet/package.json index 6eb2e5709a2..ce535734133 100644 --- a/yarn-project/cli-wallet/package.json +++ b/yarn-project/cli-wallet/package.json @@ -25,7 +25,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/cli/package.json b/yarn-project/cli/package.json index 91c1e004912..cc2407f5ee5 100644 --- a/yarn-project/cli/package.json +++ b/yarn-project/cli/package.json @@ -27,7 +27,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/entrypoints/package.json b/yarn-project/entrypoints/package.json index e498406564d..0614aa497f9 100644 --- a/yarn-project/entrypoints/package.json +++ b/yarn-project/entrypoints/package.json @@ -22,7 +22,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/epoch-cache/package.json b/yarn-project/epoch-cache/package.json index e8cb28e3330..346af28de71 100644 --- a/yarn-project/epoch-cache/package.json +++ b/yarn-project/epoch-cache/package.json @@ -22,7 +22,7 @@ "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "start:dev": "tsc-watch -p tsconfig.json --onSuccess 'yarn start'", "start": "node ./dest/index.js", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/ethereum/package.json b/yarn-project/ethereum/package.json index 1a5d81a38a8..6a4cc9df32a 100644 --- a/yarn-project/ethereum/package.json +++ b/yarn-project/ethereum/package.json @@ -23,7 +23,7 @@ "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "start:dev": "tsc-watch -p tsconfig.json --onSuccess 'yarn start'", "start": "node ./dest/index.js", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/foundation/package.json b/yarn-project/foundation/package.json index 5f64bf309b9..7376d0c2c5c 100644 --- a/yarn-project/foundation/package.json +++ b/yarn-project/foundation/package.json @@ -60,7 +60,7 @@ "generate": "true", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/key-store/package.json b/yarn-project/key-store/package.json index 1cf00ec5a75..faa128e9ae1 100644 --- a/yarn-project/key-store/package.json +++ b/yarn-project/key-store/package.json @@ -16,7 +16,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/merkle-tree/package.json b/yarn-project/merkle-tree/package.json index 022f1c210ab..042840916c4 100644 --- a/yarn-project/merkle-tree/package.json +++ b/yarn-project/merkle-tree/package.json @@ -16,7 +16,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json", diff --git a/yarn-project/noir-contracts.js/package.json b/yarn-project/noir-contracts.js/package.json index 4cb920b22f9..f37735e091b 100644 --- a/yarn-project/noir-contracts.js/package.json +++ b/yarn-project/noir-contracts.js/package.json @@ -13,7 +13,7 @@ "clean": "rm -rf ./dest .tsbuildinfo ./artifacts ./src ./codegenCache.json", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8", + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}", "generate": "yarn generate:noir-contracts", "generate:noir-contracts": "./scripts/generate-types.sh && run -T prettier -w ./src --loglevel warn" }, diff --git a/yarn-project/noir-protocol-circuits-types/package.json b/yarn-project/noir-protocol-circuits-types/package.json index b5503ab9b33..46b3671294b 100644 --- a/yarn-project/noir-protocol-circuits-types/package.json +++ b/yarn-project/noir-protocol-circuits-types/package.json @@ -24,7 +24,7 @@ "generate:vk-hashes": "node --no-warnings --loader ts-node/esm src/scripts/generate_vk_hashes.ts", "generate:noir-circuits": "node --no-warnings --loader ts-node/esm src/scripts/generate_ts_from_abi.ts && run -T prettier -w ./src/types", "generate:reset-data": "node --no-warnings --loader ts-node/esm src/scripts/generate_private_kernel_reset_data.ts && run -T prettier -w src/private_kernel_reset_data.ts", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8", + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}", "codegen": "yarn noir-codegen", "build:dev": "tsc -b --watch" }, diff --git a/yarn-project/p2p-bootstrap/package.json b/yarn-project/p2p-bootstrap/package.json index 3ee6e519217..51671674183 100644 --- a/yarn-project/p2p-bootstrap/package.json +++ b/yarn-project/p2p-bootstrap/package.json @@ -18,7 +18,7 @@ "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "start:dev": "tsc-watch -p tsconfig.json --onSuccess 'yarn start'", "start": "node ./dest/index.js", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/p2p/package.json b/yarn-project/p2p/package.json index 8524c363ee5..ab8a315ff48 100644 --- a/yarn-project/p2p/package.json +++ b/yarn-project/p2p/package.json @@ -22,7 +22,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8", + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}", "start": "node ./dest", "start:dev": "tsc-watch -p tsconfig.json --onSuccess 'yarn start'" }, diff --git a/yarn-project/package.common.json b/yarn-project/package.common.json index 270ef2d8a8a..fec3a9a873c 100644 --- a/yarn-project/package.common.json +++ b/yarn-project/package.common.json @@ -5,7 +5,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "engines": { "node": ">=18" diff --git a/yarn-project/proof-verifier/package.json b/yarn-project/proof-verifier/package.json index 31d2a9d4185..97cae4df78d 100644 --- a/yarn-project/proof-verifier/package.json +++ b/yarn-project/proof-verifier/package.json @@ -11,7 +11,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "engines": { "node": ">=18" diff --git a/yarn-project/protocol-contracts/package.json b/yarn-project/protocol-contracts/package.json index 05001587256..f555a517fa0 100644 --- a/yarn-project/protocol-contracts/package.json +++ b/yarn-project/protocol-contracts/package.json @@ -29,7 +29,7 @@ "clean": "rm -rf ./dest .tsbuildinfo ./artifacts", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json", diff --git a/yarn-project/prover-node/package.json b/yarn-project/prover-node/package.json index 9602fd7c5c8..938fec1bf7b 100644 --- a/yarn-project/prover-node/package.json +++ b/yarn-project/prover-node/package.json @@ -16,7 +16,7 @@ "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "bb": "node --no-warnings ./dest/bb/index.js", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "jest": { "moduleNameMapper": { diff --git a/yarn-project/pxe/package.json b/yarn-project/pxe/package.json index d955b4e0ce8..716483f57e1 100644 --- a/yarn-project/pxe/package.json +++ b/yarn-project/pxe/package.json @@ -23,7 +23,7 @@ "clean": "rm -rf ./dest .tsbuildinfo ./src/config/package_info.ts", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8", + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}", "start": "LOG_LEVEL=${LOG_LEVEL:-debug} && node ./dest/bin/index.js", "generate": "node ./scripts/generate_package_info.js" }, diff --git a/yarn-project/scripts/package.json b/yarn-project/scripts/package.json index f5c97d1fb85..3fc80b2aec7 100644 --- a/yarn-project/scripts/package.json +++ b/yarn-project/scripts/package.json @@ -18,7 +18,7 @@ "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "start:dev": "tsc-watch -p tsconfig.json --onSuccess 'yarn start'", "start": "node ./dest/index.js", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8", + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}", "generate:noir-circuits": "echo Noop", "generate:noir-contracts": "echo Noop", "generate:l1-contracts": "echo Noop", diff --git a/yarn-project/sequencer-client/package.json b/yarn-project/sequencer-client/package.json index ff28366a5f8..1ad914e4f81 100644 --- a/yarn-project/sequencer-client/package.json +++ b/yarn-project/sequencer-client/package.json @@ -22,7 +22,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8", + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}", "test:integration": "concurrently -k -s first -c reset,dim -n test,anvil \"yarn test:integration:run\" \"anvil\"", "test:integration:run": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --no-cache --config jest.integration.config.json" }, diff --git a/yarn-project/simulator/package.json b/yarn-project/simulator/package.json index 77bf3161e02..487bd0ff5ff 100644 --- a/yarn-project/simulator/package.json +++ b/yarn-project/simulator/package.json @@ -22,7 +22,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/telemetry-client/package.json b/yarn-project/telemetry-client/package.json index fe08f53592a..6cad929392f 100644 --- a/yarn-project/telemetry-client/package.json +++ b/yarn-project/telemetry-client/package.json @@ -16,7 +16,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "engines": { "node": ">=18" diff --git a/yarn-project/txe/package.json b/yarn-project/txe/package.json index 06805e08b9d..d17b71b0533 100644 --- a/yarn-project/txe/package.json +++ b/yarn-project/txe/package.json @@ -17,7 +17,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8", + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}", "dev": "LOG_LEVEL=debug node ./dest/bin/index.js", "start": "node ./dest/bin/index.js" }, diff --git a/yarn-project/types/package.json b/yarn-project/types/package.json index e7c108644b2..3a2206e7957 100644 --- a/yarn-project/types/package.json +++ b/yarn-project/types/package.json @@ -18,7 +18,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8", + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}", "generate": "yarn generate:noir-contracts", "generate:noir-contracts": "./scripts/copy-contracts.sh" }, diff --git a/yarn-project/validator-client/package.json b/yarn-project/validator-client/package.json index 8861df3557c..8970687e0dc 100644 --- a/yarn-project/validator-client/package.json +++ b/yarn-project/validator-client/package.json @@ -22,7 +22,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json" diff --git a/yarn-project/world-state/package.json b/yarn-project/world-state/package.json index 57e3dc24993..b3bdd5346c6 100644 --- a/yarn-project/world-state/package.json +++ b/yarn-project/world-state/package.json @@ -22,7 +22,7 @@ "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "generate": "mkdir -p build && cp -v ../../barretenberg/cpp/build-pic/lib/world_state_napi.node build", - "test": "HARDWARE_CONCURRENCY=16 RAYON_NUM_THREADS=4 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=8" + "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" }, "inherits": [ "../package.common.json",