From 64c08b647cf1d4dddaa437eae96f579d31079e2e Mon Sep 17 00:00:00 2001 From: spypsy Date: Mon, 8 Jan 2024 14:59:37 +0000 Subject: [PATCH 01/50] WIP 'aztec' CLI & renaming sandbox --- .circleci/config.yml | 20 +-- README.md | 2 +- boxes/yarn.lock | 8 - build_manifest.yml | 10 +- .../functions/ecdsa_secp256k1_verify.md | 1 - .../docs/reference/NoirJS/noir_js/index.md | 2 +- .../src/archiver/archiver_http_server.ts | 38 +++++ yarn-project/aztec-node/package.json | 1 + .../aztec-node/src/aztec-node/config.ts | 4 + .../aztec-node/src/aztec-node/server.ts | 13 +- yarn-project/aztec-node/tsconfig.json | 9 +- yarn-project/aztec.js/src/index.ts | 3 +- .../src/rpc_clients/archiver_client.ts | 37 +++++ .../src/rpc_clients}/aztec_node_client.ts | 23 +-- .../aztec.js/src/rpc_clients/index.ts | 3 + .../src/{ => rpc_clients}/pxe_client.ts | 2 - yarn-project/aztec.js/src/sandbox/index.ts | 2 +- .../aztec.js/src/utils/l1_contracts.ts | 2 +- .../{aztec-sandbox => aztec}/.eslintrc.cjs | 0 .../{aztec-sandbox => aztec}/.gitignore | 0 .../{aztec-sandbox => aztec}/Dockerfile | 0 .../{aztec-sandbox => aztec}/README.md | 0 .../docker-compose.yml | 0 .../{aztec-sandbox => aztec}/package.json | 9 +- yarn-project/aztec/src/bin/cli.ts | 150 ++++++++++++++++++ .../{aztec-sandbox => aztec}/src/bin/index.ts | 0 .../src/examples/token.ts | 0 .../src/examples/util.ts | 0 .../{aztec-sandbox => aztec}/src/index.ts | 0 .../{aztec-sandbox => aztec}/src/logging.ts | 0 .../{aztec-sandbox => aztec}/src/sandbox.ts | 0 .../{aztec-sandbox => aztec}/src/server.ts | 0 .../{aztec-sandbox => aztec}/src/splash.ts | 0 .../{aztec-sandbox => aztec}/tsconfig.json | 0 yarn-project/p2p-bootstrap/terraform/main.tf | 2 +- yarn-project/p2p/tsconfig.json | 4 +- yarn-project/package.json | 4 +- yarn-project/pxe/package.json | 1 + yarn-project/pxe/src/bin/index.ts | 2 +- yarn-project/pxe/tsconfig.json | 3 + .../types/src/aztec_node/rpc/index.ts | 1 - yarn-project/types/src/contract_data.ts | 17 ++ yarn-project/types/src/index.ts | 10 +- yarn-project/types/src/l1_to_l2_message.ts | 9 ++ .../types/src/logs/l2_block_l2_logs.ts | 18 +++ yarn-project/yarn.lock | 11 ++ 46 files changed, 363 insertions(+), 58 deletions(-) create mode 100644 yarn-project/archiver/src/archiver/archiver_http_server.ts create mode 100644 yarn-project/aztec.js/src/rpc_clients/archiver_client.ts rename yarn-project/{types/src/aztec_node/rpc => aztec.js/src/rpc_clients}/aztec_node_client.ts (68%) create mode 100644 yarn-project/aztec.js/src/rpc_clients/index.ts rename yarn-project/aztec.js/src/{ => rpc_clients}/pxe_client.ts (94%) rename yarn-project/{aztec-sandbox => aztec}/.eslintrc.cjs (100%) rename yarn-project/{aztec-sandbox => aztec}/.gitignore (100%) rename yarn-project/{aztec-sandbox => aztec}/Dockerfile (100%) rename yarn-project/{aztec-sandbox => aztec}/README.md (100%) rename yarn-project/{aztec-sandbox => aztec}/docker-compose.yml (100%) rename yarn-project/{aztec-sandbox => aztec}/package.json (92%) create mode 100644 yarn-project/aztec/src/bin/cli.ts rename yarn-project/{aztec-sandbox => aztec}/src/bin/index.ts (100%) rename yarn-project/{aztec-sandbox => aztec}/src/examples/token.ts (100%) rename yarn-project/{aztec-sandbox => aztec}/src/examples/util.ts (100%) rename yarn-project/{aztec-sandbox => aztec}/src/index.ts (100%) rename yarn-project/{aztec-sandbox => aztec}/src/logging.ts (100%) rename yarn-project/{aztec-sandbox => aztec}/src/sandbox.ts (100%) rename yarn-project/{aztec-sandbox => aztec}/src/server.ts (100%) rename yarn-project/{aztec-sandbox => aztec}/src/splash.ts (100%) rename yarn-project/{aztec-sandbox => aztec}/tsconfig.json (100%) delete mode 100644 yarn-project/types/src/aztec_node/rpc/index.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index b049069c8c0..04aa129c6a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -405,7 +405,7 @@ jobs: name: Test command: cond_spot_run_container yarn-project 64 test | add_timestamps - aztec-sandbox: + aztec-package: machine: image: ubuntu-2204:2023.07.2 resource_class: large @@ -414,7 +414,7 @@ jobs: - *setup_env - run: name: "Build and test" - command: build aztec-sandbox + command: build aztec cli: machine: @@ -965,7 +965,7 @@ jobs: command: | should_release || exit 0 deploy_dockerhub noir - deploy_dockerhub aztec-sandbox + deploy_dockerhub aztec deploy_dockerhub cli deploy_dockerhub aztec-faucet deploy_dockerhub mainnet-fork @@ -1007,7 +1007,7 @@ jobs: # Export variables for Terraform. export TF_VAR_BOOTNODE_1_PRIVATE_KEY=$BOOTNODE_1_PRIVATE_KEY export TF_VAR_BOOTNODE_2_PRIVATE_KEY=$BOOTNODE_2_PRIVATE_KEY - deploy_terraform_services yarn-project/p2p-bootstrap aztec-sandbox + deploy_terraform_services yarn-project/p2p-bootstrap aztec - run: name: "Deploy Aztec Nodes to AWS" command: | @@ -1021,16 +1021,16 @@ jobs: # Check if l1-contracts have changed if [ "$CONTRACTS_DEPLOYED" -eq 1 ]; then echo "Contracts have changed, taint nodes to force redeploy.." - deploy_terraform_services yarn-project/aztec-node aztec-sandbox aztec-node "aws_ecs_task_definition.aztec-node[0],aws_ecs_task_definition.aztec-node[1]" + deploy_terraform_services yarn-project/aztec-node aztec aztec-node "aws_ecs_task_definition.aztec-node[0],aws_ecs_task_definition.aztec-node[1]" else - deploy_terraform_services yarn-project/aztec-node aztec-sandbox + deploy_terraform_services yarn-project/aztec-node aztec fi - run: name: "Deploy Aztec Faucet to AWS" command: | should_deploy 0 || exit 0 export TF_VAR_FAUCET_PRIVATE_KEY=$FAUCET_PRIVATE_KEY - deploy_terraform_services yarn-project/aztec-faucet aztec-sandbox + deploy_terraform_services yarn-project/aztec-faucet aztec # Repeatable config for defining the workflow below. defaults: &defaults @@ -1152,14 +1152,14 @@ workflows: - build-docs: *defaults_yarn_project # Artifacts - - aztec-sandbox: *defaults_yarn_project_prod + - aztec-package: *defaults_yarn_project_prod - cli: *defaults_yarn_project_prod - aztec-faucet: *defaults_yarn_project_prod # Boxes. - boxes: requires: - - aztec-sandbox + - aztec-package <<: *defaults - boxes-blank: requires: @@ -1178,7 +1178,7 @@ workflows: - e2e-join: requires: - end-to-end - - aztec-sandbox + - aztec-package - cli <<: *defaults - e2e-2-pxes: *e2e_test diff --git a/README.md b/README.md index 8679737846a..f4e8b83a075 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ All the packages that make up [Aztec](https://docs.aztec.network). ## Popular packages - [Aztec.nr](./yarn-project/aztec-nr/): A [Noir](https://noir-lang.org) framework for smart contracts on Aztec. -- [Aztec Sandbox](./yarn-project/aztec-sandbox/): A package for setting up a local dev net, including a local Ethereum network, deployed rollup contracts and Aztec execution environment. +- [Aztec](./yarn-project/aztec/): A package for starting up local dev net modules, including a local 'sandbox' devnet, an Ethereum network, deployed rollup contracts and Aztec execution environment. - [Aztec.js](./yarn-project/aztec.js/): A tool for interacting with the Aztec network. It communicates via the [Private Execution Environment (PXE)](./yarn-project/pxe/). - [Example contracts](./yarn-project/noir-contracts/): Example contracts for the Aztec network, written in Noir. - [End to end tests](./yarn-project/end-to-end/): Integration tests written in Typescript--a good reference for how to use the packages for specific tasks. diff --git a/boxes/yarn.lock b/boxes/yarn.lock index 62fd6f8acf2..53baa30c52b 100644 --- a/boxes/yarn.lock +++ b/boxes/yarn.lock @@ -58,7 +58,6 @@ __metadata: "@aztec/types": "workspace:^" lodash.every: "npm:^4.6.0" lodash.partition: "npm:^4.6.0" - lodash.zip: "npm:^4.2.0" tslib: "npm:^2.4.0" languageName: node linkType: soft @@ -9472,13 +9471,6 @@ __metadata: languageName: node linkType: hard -"lodash.zip@npm:^4.2.0": - version: 4.2.0 - resolution: "lodash.zip@npm:4.2.0" - checksum: e596da80a6138e369998b50c78b51ed6cf984b4f239e59056aa18dca5972a213c491c511caf5888a2dec603c67265caf942099bec554a86a5c7ff1937d57f0e4 - languageName: node - linkType: hard - "lodash@npm:^4.17.11, lodash@npm:^4.17.15, lodash@npm:^4.17.21": version: 4.17.21 resolution: "lodash@npm:4.17.21" diff --git a/build_manifest.yml b/build_manifest.yml index be3cf243273..f43200d6dc7 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -146,9 +146,9 @@ yarn-project-prod: - yarn-project multiarch: buildx -aztec-sandbox: +aztec: buildDir: yarn-project - projectDir: yarn-project/aztec-sandbox + projectDir: yarn-project/aztec dependencies: - yarn-project-prod multiarch: buildx @@ -169,10 +169,10 @@ cli: boxes: buildDir: boxes dependencies: - - aztec-sandbox + - aztec - noir runDependencies: - - aztec-sandbox + - aztec end-to-end: buildDir: yarn-project @@ -180,7 +180,7 @@ end-to-end: dependencies: - yarn-project runDependencies: - - aztec-sandbox + - aztec mainnet-fork: buildDir: iac/mainnet-fork diff --git a/noir/docs/docs/reference/NoirJS/noir_js/functions/ecdsa_secp256k1_verify.md b/noir/docs/docs/reference/NoirJS/noir_js/functions/ecdsa_secp256k1_verify.md index 0ba5783f0d5..5e3cd53e9d3 100644 --- a/noir/docs/docs/reference/NoirJS/noir_js/functions/ecdsa_secp256k1_verify.md +++ b/noir/docs/docs/reference/NoirJS/noir_js/functions/ecdsa_secp256k1_verify.md @@ -8,7 +8,6 @@ ecdsa_secp256k1_verify( signature): boolean ``` -Calculates the Blake2s256 hash of the input bytes and represents these as a single field element. Verifies a ECDSA signature over the secp256k1 curve. ## Parameters diff --git a/noir/docs/docs/reference/NoirJS/noir_js/index.md b/noir/docs/docs/reference/NoirJS/noir_js/index.md index 8b9e35bc9a1..d600e21b299 100644 --- a/noir/docs/docs/reference/NoirJS/noir_js/index.md +++ b/noir/docs/docs/reference/NoirJS/noir_js/index.md @@ -26,7 +26,7 @@ | :------ | :------ | | [and](functions/and.md) | Performs a bitwise AND operation between `lhs` and `rhs` | | [blake2s256](functions/blake2s256.md) | Calculates the Blake2s256 hash of the input bytes | -| [ecdsa\_secp256k1\_verify](functions/ecdsa_secp256k1_verify.md) | Calculates the Blake2s256 hash of the input bytes and represents these as a single field element. | +| [ecdsa\_secp256k1\_verify](functions/ecdsa_secp256k1_verify.md) | Verifies a ECDSA signature over the secp256k1 curve. | | [ecdsa\_secp256r1\_verify](functions/ecdsa_secp256r1_verify.md) | Verifies a ECDSA signature over the secp256r1 curve. | | [keccak256](functions/keccak256.md) | Calculates the Keccak256 hash of the input bytes | | [sha256](functions/sha256.md) | Calculates the SHA256 hash of the input bytes | diff --git a/yarn-project/archiver/src/archiver/archiver_http_server.ts b/yarn-project/archiver/src/archiver/archiver_http_server.ts new file mode 100644 index 00000000000..beeabfdc4f7 --- /dev/null +++ b/yarn-project/archiver/src/archiver/archiver_http_server.ts @@ -0,0 +1,38 @@ +import { EthAddress, Fr } from '@aztec/circuits.js'; +import { JsonRpcServer } from '@aztec/foundation/json-rpc/server'; +import { + ContractData, + EncodedContractFunction, + ExtendedContractData, + ExtendedUnencryptedL2Log, + L1ToL2Message, + L2Block, + L2BlockL2Logs, +} from '@aztec/types'; + +import { Archiver } from './archiver.js'; + +/** + * Wrap an Archiver instance with a JSON RPC HTTP server. + * @param archiverService - The Archiver instance + * @returns An JSON-RPC HTTP server + */ +export function createArchiverRpcServer(archiverService: Archiver): JsonRpcServer { + return new JsonRpcServer( + archiverService, + { + ContractData, + EncodedContractFunction, + EthAddress, + ExtendedContractData, + ExtendedUnencryptedL2Log, + Fr, + L1ToL2Message, + L2Block, + L2BlockL2Logs, + }, + {}, + false, + ['start', 'stop'], + ); +} diff --git a/yarn-project/aztec-node/package.json b/yarn-project/aztec-node/package.json index 6dd220c05b7..982ec8f925a 100644 --- a/yarn-project/aztec-node/package.json +++ b/yarn-project/aztec-node/package.json @@ -34,6 +34,7 @@ }, "dependencies": { "@aztec/archiver": "workspace:^", + "@aztec/aztec.js": "workspace:^", "@aztec/circuits.js": "workspace:^", "@aztec/ethereum": "workspace:^", "@aztec/foundation": "workspace:^", diff --git a/yarn-project/aztec-node/src/aztec-node/config.ts b/yarn-project/aztec-node/src/aztec-node/config.ts index 185bd3d8b6b..73f14115804 100644 --- a/yarn-project/aztec-node/src/aztec-node/config.ts +++ b/yarn-project/aztec-node/src/aztec-node/config.ts @@ -11,6 +11,9 @@ export type AztecNodeConfig = ArchiverConfig & P2PConfig & { /** Whether the sequencer is disabled for this node. */ disableSequencer: boolean; + + /** A URL for an archiver service that the node will use. */ + archiverUrl?: string; }; /** @@ -25,6 +28,7 @@ export function getConfigEnvVars(): AztecNodeConfig { ...getP2PConfigEnvVars(), ...getWorldStateVars(), disableSequencer: !!SEQ_DISABLED, + archiverUrl: process.env.ARCHIVER_URL, }; return allEnvVars; diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 36d19ed675f..8f3777e51ef 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -1,4 +1,5 @@ import { Archiver, LMDBArchiverStore } from '@aztec/archiver'; +import { createArchiverClient } from '@aztec/aztec.js'; import { ARCHIVE_HEIGHT, BlockHeader, @@ -25,6 +26,7 @@ import { getGlobalVariableBuilder, } from '@aztec/sequencer-client'; import { + ArchiveSource, AztecNode, ContractData, ContractDataSource, @@ -109,9 +111,14 @@ export class AztecNodeService implements AztecNode { const store = await AztecLmdbStore.create(config.l1Contracts.rollupAddress, config.dataDirectory); const [nodeDb, worldStateDb] = await openDb(config, log); - // first create and sync the archiver - const archiverStore = new LMDBArchiverStore(nodeDb, config.maxLogs); - const archiver = await Archiver.createAndSync(config, archiverStore, true); + let archiver: ArchiveSource; + if (!config.archiverUrl) { + // first create and sync the archiver + const archiverStore = new LMDBArchiverStore(nodeDb, config.maxLogs); + archiver = await Archiver.createAndSync(config, archiverStore, true); + } else { + archiver = createArchiverClient(config.archiverUrl); + } // we identify the P2P transaction protocol by using the rollup contract address. // this may well change in future diff --git a/yarn-project/aztec-node/tsconfig.json b/yarn-project/aztec-node/tsconfig.json index 3c938379585..8956ac80f70 100644 --- a/yarn-project/aztec-node/tsconfig.json +++ b/yarn-project/aztec-node/tsconfig.json @@ -9,6 +9,9 @@ { "path": "../archiver" }, + { + "path": "../aztec.js" + }, { "path": "../circuits.js" }, @@ -18,6 +21,9 @@ { "path": "../foundation" }, + { + "path": "../kv-store" + }, { "path": "../l1-artifacts" }, @@ -35,9 +41,6 @@ }, { "path": "../world-state" - }, - { - "path": "../kv-store" } ], "include": ["src"] diff --git a/yarn-project/aztec.js/src/index.ts b/yarn-project/aztec.js/src/index.ts index 6ad796595e4..15d8857cbc2 100644 --- a/yarn-project/aztec.js/src/index.ts +++ b/yarn-project/aztec.js/src/index.ts @@ -46,7 +46,7 @@ export { computeAuthWitMessageHash, } from './utils/index.js'; -export { createPXEClient } from './pxe_client.js'; +export { createPXEClient, createAztecNodeClient, createArchiverClient } from './rpc_clients/index.js'; export { CompleteAddress, AuthWitnessProvider } from './account/index.js'; @@ -112,7 +112,6 @@ export { TxReceipt, TxStatus, UnencryptedL2Log, - createAztecNodeClient, emptyFunctionCall, merkleTreeIds, mockTx, diff --git a/yarn-project/aztec.js/src/rpc_clients/archiver_client.ts b/yarn-project/aztec.js/src/rpc_clients/archiver_client.ts new file mode 100644 index 00000000000..10accb355a7 --- /dev/null +++ b/yarn-project/aztec.js/src/rpc_clients/archiver_client.ts @@ -0,0 +1,37 @@ +import { EthAddress, Fr } from '@aztec/circuits.js'; +import { createJsonRpcClient, makeFetch } from '@aztec/foundation/json-rpc/client'; +import { + ContractData, + ContractDataSource, + EncodedContractFunction, + ExtendedContractData, + ExtendedUnencryptedL2Log, + L1ToL2Message, + L1ToL2MessageSource, + L2Block, + L2BlockL2Logs, + L2BlockSource, + L2LogsSource, +} from '@aztec/types'; + +export const createArchiverClient = ( + url: string, + fetch = makeFetch([1, 2, 3], true), +): L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource => + createJsonRpcClient( + url, + { + ContractData, + EncodedContractFunction, + EthAddress, + ExtendedContractData, + ExtendedUnencryptedL2Log, + Fr, + L1ToL2Message, + L2Block, + L2BlockL2Logs, + }, + {}, + false, + fetch, + ); diff --git a/yarn-project/types/src/aztec_node/rpc/aztec_node_client.ts b/yarn-project/aztec.js/src/rpc_clients/aztec_node_client.ts similarity index 68% rename from yarn-project/types/src/aztec_node/rpc/aztec_node_client.ts rename to yarn-project/aztec.js/src/rpc_clients/aztec_node_client.ts index 184801cd9bd..4f2f702c21c 100644 --- a/yarn-project/types/src/aztec_node/rpc/aztec_node_client.ts +++ b/yarn-project/aztec.js/src/rpc_clients/aztec_node_client.ts @@ -3,15 +3,20 @@ import { AztecAddress } from '@aztec/foundation/aztec-address'; import { EthAddress } from '@aztec/foundation/eth-address'; import { Fr } from '@aztec/foundation/fields'; import { createJsonRpcClient, defaultFetch } from '@aztec/foundation/json-rpc/client'; - -import { ContractData, ExtendedContractData } from '../../contract_data.js'; -import { AztecNode } from '../../interfaces/index.js'; -import { L1ToL2MessageAndIndex } from '../../l1_to_l2_message.js'; -import { L2Block } from '../../l2_block.js'; -import { L2Tx } from '../../l2_tx.js'; -import { ExtendedUnencryptedL2Log, L2BlockL2Logs, LogId } from '../../logs/index.js'; -import { SiblingPath } from '../../sibling_path.js'; -import { Tx, TxHash } from '../../tx/index.js'; +import { + AztecNode, + ContractData, + ExtendedContractData, + ExtendedUnencryptedL2Log, + L1ToL2MessageAndIndex, + L2Block, + L2BlockL2Logs, + L2Tx, + LogId, + SiblingPath, + Tx, + TxHash, +} from '@aztec/types'; /** * Creates a JSON-RPC client to remotely talk to an Aztec Node. diff --git a/yarn-project/aztec.js/src/rpc_clients/index.ts b/yarn-project/aztec.js/src/rpc_clients/index.ts new file mode 100644 index 00000000000..43b4ba5cd33 --- /dev/null +++ b/yarn-project/aztec.js/src/rpc_clients/index.ts @@ -0,0 +1,3 @@ +export * from './pxe_client.js'; +export * from './aztec_node_client.js'; +export * from './archiver_client.js'; diff --git a/yarn-project/aztec.js/src/pxe_client.ts b/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts similarity index 94% rename from yarn-project/aztec.js/src/pxe_client.ts rename to yarn-project/aztec.js/src/rpc_clients/pxe_client.ts index 5e7cd3a3822..2b3bf6fcc6a 100644 --- a/yarn-project/aztec.js/src/pxe_client.ts +++ b/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts @@ -26,8 +26,6 @@ import { TxReceipt, } from '@aztec/types'; -export { makeFetch } from '@aztec/foundation/json-rpc/client'; - /** * Creates a JSON-RPC client to remotely talk to PXE. * @param url - The URL of the PXE. diff --git a/yarn-project/aztec.js/src/sandbox/index.ts b/yarn-project/aztec.js/src/sandbox/index.ts index 79f9f068b1d..3b050ea9a11 100644 --- a/yarn-project/aztec.js/src/sandbox/index.ts +++ b/yarn-project/aztec.js/src/sandbox/index.ts @@ -3,7 +3,7 @@ import { sleep } from '@aztec/foundation/sleep'; import { PXE } from '@aztec/types'; import { getSchnorrAccount } from '../account_manager/index.js'; -import { createPXEClient } from '../pxe_client.js'; +import { createPXEClient } from '../rpc_clients/index.js'; import { AccountWalletWithPrivateKey } from '../wallet/index.js'; export const INITIAL_SANDBOX_ENCRYPTION_KEYS = [ diff --git a/yarn-project/aztec.js/src/utils/l1_contracts.ts b/yarn-project/aztec.js/src/utils/l1_contracts.ts index 7275756a579..3b4b05ee8e7 100644 --- a/yarn-project/aztec.js/src/utils/l1_contracts.ts +++ b/yarn-project/aztec.js/src/utils/l1_contracts.ts @@ -1,7 +1,7 @@ import { L1ContractAddresses } from '@aztec/ethereum'; import { retryUntil } from '@aztec/foundation/retry'; -import { createPXEClient } from '../pxe_client.js'; +import { createPXEClient } from '../rpc_clients/index.js'; export const getL1ContractAddresses = async (url: string): Promise => { const pxeClient = createPXEClient(url); diff --git a/yarn-project/aztec-sandbox/.eslintrc.cjs b/yarn-project/aztec/.eslintrc.cjs similarity index 100% rename from yarn-project/aztec-sandbox/.eslintrc.cjs rename to yarn-project/aztec/.eslintrc.cjs diff --git a/yarn-project/aztec-sandbox/.gitignore b/yarn-project/aztec/.gitignore similarity index 100% rename from yarn-project/aztec-sandbox/.gitignore rename to yarn-project/aztec/.gitignore diff --git a/yarn-project/aztec-sandbox/Dockerfile b/yarn-project/aztec/Dockerfile similarity index 100% rename from yarn-project/aztec-sandbox/Dockerfile rename to yarn-project/aztec/Dockerfile diff --git a/yarn-project/aztec-sandbox/README.md b/yarn-project/aztec/README.md similarity index 100% rename from yarn-project/aztec-sandbox/README.md rename to yarn-project/aztec/README.md diff --git a/yarn-project/aztec-sandbox/docker-compose.yml b/yarn-project/aztec/docker-compose.yml similarity index 100% rename from yarn-project/aztec-sandbox/docker-compose.yml rename to yarn-project/aztec/docker-compose.yml diff --git a/yarn-project/aztec-sandbox/package.json b/yarn-project/aztec/package.json similarity index 92% rename from yarn-project/aztec-sandbox/package.json rename to yarn-project/aztec/package.json index 0739a8d9cc4..2142886e020 100644 --- a/yarn-project/aztec-sandbox/package.json +++ b/yarn-project/aztec/package.json @@ -1,5 +1,5 @@ { - "name": "@aztec/aztec-sandbox", + "name": "@aztec/aztec", "version": "0.1.0", "type": "module", "exports": { @@ -10,12 +10,13 @@ "entryPoints": [ "./src/index.ts" ], - "name": "Sandbox", + "name": "Aztec Packages", "tsconfig": "./tsconfig.json" }, "scripts": { "build": "yarn clean && tsc -b", "start": "node --no-warnings ./dest/bin", + "start:cli": "node --no-warnings ./dest/bin/cli.js", "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", @@ -39,8 +40,10 @@ "@aztec/pxe": "workspace:^", "@aztec/types": "workspace:^", "abitype": "^0.8.11", + "commander": "^11.1.0", "koa": "^2.14.2", "koa-router": "^12.0.0", + "lodash.pick": "^4.4.0", "viem": "^1.2.5", "winston": "^3.10.0", "winston-daily-rotate-file": "^4.7.1" @@ -71,4 +74,4 @@ "engines": { "node": ">=18" } -} +} \ No newline at end of file diff --git a/yarn-project/aztec/src/bin/cli.ts b/yarn-project/aztec/src/bin/cli.ts new file mode 100644 index 00000000000..90bd71def92 --- /dev/null +++ b/yarn-project/aztec/src/bin/cli.ts @@ -0,0 +1,150 @@ +import { createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; +import { fileURLToPath } from '@aztec/aztec.js'; +import { DebugLogger, LogFn, createConsoleLogger, createDebugLogger } from '@aztec/foundation/log'; +import { createPXERpcServer, getPXEServiceConfig } from '@aztec/pxe'; + +import { Command } from 'commander'; +import { readFileSync } from 'fs'; +import pick from 'lodash.pick'; +import { dirname, resolve } from 'path'; + +import { MNEMONIC, createAztecNode, createAztecPXE, createSandbox } from '../sandbox.js'; +import { startHttpRpcServer } from '../server.js'; + +const { AZTEC_NODE_PORT = '80', PXE_PORT = '79' } = process.env; + +const debugLogger = createDebugLogger('aztec:cli'); + +const installSignalHandlers = (logFn: LogFn, cb?: Array<() => Promise>) => { + const shutdown = async () => { + logFn('Shutting down...'); + if (cb) { + await Promise.all(cb); + } + process.exit(0); + }; + process.removeAllListeners('SIGINT'); + process.removeAllListeners('SIGTERM'); + process.once('SIGINT', shutdown); + process.once('SIGTERM', shutdown); +}; + +/** + * Parses a string of options into a key-value map. + * @param options - String of options in the format "option1=value1,option2=value2". + * @returns Key-value map of options. + */ +const parseModuleOptions = (options: string): Record => { + const optionsArray = options.split(','); + return optionsArray.reduce((acc, option) => { + const [key, value] = option.split('='); + return { ...acc, [key]: value }; + }, {}); +}; + +/** + * Returns commander program that defines the 'aztec' command line interface. + * @param userLog - log function for logging user output. + * @param debugLogger - logger for logging debug messages. + */ +export function getProgram(userLog: LogFn /*debugLogger: DebugLogger*/): Command { + const program = new Command(); + + const packageJsonPath = resolve(dirname(fileURLToPath(import.meta.url)), '../../package.json'); + const cliVersion: string = JSON.parse(readFileSync(packageJsonPath).toString()).version; + + program.name('aztec').description('Aztec command line interface').version(cliVersion); + + // Start complete Sandbox. + program + .command('sandbox', 'Starts Aztec sandbox') + .option('-p, --port ', 'Port to run sandbox on', AZTEC_NODE_PORT) + .option('-pp, --pxe-port ', 'Port to run PXE on', PXE_PORT) + .action(async options => { + const { node, pxe, stop } = await createSandbox(); + installSignalHandlers(stop); + + // Start Node and PXE JSON-RPC servers + startHttpRpcServer(node, createAztecNodeRpcServer, options.port); + userLog(`Aztec Node JSON-RPC Server listening on port ${options.port}`); + if (options.pxePort) { + startHttpRpcServer(pxe, createPXERpcServer, PXE_PORT); + userLog(`PXE JSON-RPC Server listening on port ${PXE_PORT}`); + } + }); + + // Start Aztec modules with options + program + .command('start', 'Starts Aztec modules') + .option('-n, --node [options]', 'Starts Aztec Node with options (e.g. "option1=value1,option2=value2")') + .option('-p, --pxe [options]', 'Starts Aztec PXE with options (e.g. "option1=value1,option2=value2")') + .option('-a, --archiver [options]', 'Starts Aztec Archiver attached to node') + .option('-s, --sequencer [options]', 'Starts Aztec Sequencer attached to node') + .action(async options => { + // list of 'stop' functions to call when process ends + const signalHandlers: Array<() => Promise> = []; + + // Start Aztec Node + if (options.node) { + // get env vars first + const aztecNodeConfigEnvVars = getNodeConfigEnvVars(); + // get config from options + const nodeCliOptions = parseModuleOptions(options.node); + // merge env vars and cli options + const nodeConfig = pick({ ...aztecNodeConfigEnvVars, ...nodeCliOptions }); + + // if no publisher private key, then use MNEMONIC + if (nodeConfig.publisherPrivateKey === undefined) { + if (!options.archiver) { + // expect archiver url in node config + const archiverUrl = nodeCliOptions.archiverUrl; + if (!archiverUrl) { + throw new Error('Archiver Service URL is required to start Aztec Node without --archiver option'); + } + nodeConfig.archiverUrl = archiverUrl; + } + + const node = await createAztecNode(nodeConfig); + userLog(`Aztec Node JSON-RPC Server listening on port ${nodeCliOptions.port}`); + + // Create a PXE client that connects to the node + if (options.pxe) { + const pxe = await createAztecPXE(node); + signalHandlers.push(pxe.stop); + + // Start PXE JSON-RPC server + startHttpRpcServer(pxe, createPXERpcServer, options.PXE_PORT); + } + + signalHandlers.push(node.stop); + } + + if (options.pxe) { + // get env vars first + const pxeConfigEnvVars = getPXEServiceConfig(); + // get config from options + const pxeCliOptions = parseModuleOptions(options.pxe); + // merge env vars and cli options + const pxeConfig = pick({ ...pxeConfigEnvVars, ...pxeCliOptions }); + + const pxe = createPXERpcServer(pxeConfig); + // userLog(`PXE JSON-RPC Server listening on port ${pxeOptions.port}`); + } + installSignalHandlers(signalHandlers); + } + }); + return program; +} + +/** CLI main entrypoint */ +async function main() { + const userLog = createConsoleLogger(); + const program = getProgram(userLog); + await program.parseAsync(process.argv); +} + +main().catch(err => { + debugLogger(`Error in command execution`); + debugLogger(err); + process.exit(1); +}); diff --git a/yarn-project/aztec-sandbox/src/bin/index.ts b/yarn-project/aztec/src/bin/index.ts similarity index 100% rename from yarn-project/aztec-sandbox/src/bin/index.ts rename to yarn-project/aztec/src/bin/index.ts diff --git a/yarn-project/aztec-sandbox/src/examples/token.ts b/yarn-project/aztec/src/examples/token.ts similarity index 100% rename from yarn-project/aztec-sandbox/src/examples/token.ts rename to yarn-project/aztec/src/examples/token.ts diff --git a/yarn-project/aztec-sandbox/src/examples/util.ts b/yarn-project/aztec/src/examples/util.ts similarity index 100% rename from yarn-project/aztec-sandbox/src/examples/util.ts rename to yarn-project/aztec/src/examples/util.ts diff --git a/yarn-project/aztec-sandbox/src/index.ts b/yarn-project/aztec/src/index.ts similarity index 100% rename from yarn-project/aztec-sandbox/src/index.ts rename to yarn-project/aztec/src/index.ts diff --git a/yarn-project/aztec-sandbox/src/logging.ts b/yarn-project/aztec/src/logging.ts similarity index 100% rename from yarn-project/aztec-sandbox/src/logging.ts rename to yarn-project/aztec/src/logging.ts diff --git a/yarn-project/aztec-sandbox/src/sandbox.ts b/yarn-project/aztec/src/sandbox.ts similarity index 100% rename from yarn-project/aztec-sandbox/src/sandbox.ts rename to yarn-project/aztec/src/sandbox.ts diff --git a/yarn-project/aztec-sandbox/src/server.ts b/yarn-project/aztec/src/server.ts similarity index 100% rename from yarn-project/aztec-sandbox/src/server.ts rename to yarn-project/aztec/src/server.ts diff --git a/yarn-project/aztec-sandbox/src/splash.ts b/yarn-project/aztec/src/splash.ts similarity index 100% rename from yarn-project/aztec-sandbox/src/splash.ts rename to yarn-project/aztec/src/splash.ts diff --git a/yarn-project/aztec-sandbox/tsconfig.json b/yarn-project/aztec/tsconfig.json similarity index 100% rename from yarn-project/aztec-sandbox/tsconfig.json rename to yarn-project/aztec/tsconfig.json diff --git a/yarn-project/p2p-bootstrap/terraform/main.tf b/yarn-project/p2p-bootstrap/terraform/main.tf index 659133d4f42..8245a787bc5 100644 --- a/yarn-project/p2p-bootstrap/terraform/main.tf +++ b/yarn-project/p2p-bootstrap/terraform/main.tf @@ -105,7 +105,7 @@ resource "aws_ecs_task_definition" "p2p-bootstrap" { [ { "name": "${var.DEPLOY_TAG}-p2p-bootstrap-${count.index + 1}", - "image": "${var.DOCKERHUB_ACCOUNT}/aztec-sandbox:${var.DEPLOY_TAG}", + "image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}", "essential": true, "command": ["start"], "memoryReservation": 3776, diff --git a/yarn-project/p2p/tsconfig.json b/yarn-project/p2p/tsconfig.json index b780dee9f30..e934003361b 100644 --- a/yarn-project/p2p/tsconfig.json +++ b/yarn-project/p2p/tsconfig.json @@ -13,10 +13,10 @@ "path": "../foundation" }, { - "path": "../types" + "path": "../kv-store" }, { - "path": "../kv-store" + "path": "../types" } ], "include": ["src"] diff --git a/yarn-project/package.json b/yarn-project/package.json index 9769892147b..a387fc315b8 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -21,7 +21,7 @@ "aztec-faucet", "aztec-node", "pxe", - "aztec-sandbox", + "aztec", "aztec.js", "circuits.js", "cli", @@ -69,4 +69,4 @@ "@noir-lang/noir_wasm": "portal:../noir/packages/noir_wasm", "@noir-lang/noir_js": "portal:../noir/packages/noir_js" } -} +} \ No newline at end of file diff --git a/yarn-project/pxe/package.json b/yarn-project/pxe/package.json index b1892b3b1f3..b66787bc9dd 100644 --- a/yarn-project/pxe/package.json +++ b/yarn-project/pxe/package.json @@ -34,6 +34,7 @@ }, "dependencies": { "@aztec/acir-simulator": "workspace:^", + "@aztec/aztec.js": "workspace:^", "@aztec/circuits.js": "workspace:^", "@aztec/ethereum": "workspace:^", "@aztec/foundation": "workspace:^", diff --git a/yarn-project/pxe/src/bin/index.ts b/yarn-project/pxe/src/bin/index.ts index 43e49dd8271..538b482c940 100644 --- a/yarn-project/pxe/src/bin/index.ts +++ b/yarn-project/pxe/src/bin/index.ts @@ -1,7 +1,7 @@ #!/usr/bin/env -S node --no-warnings +import { createAztecNodeClient } from '@aztec/aztec.js'; import { init } from '@aztec/foundation/crypto'; import { createDebugLogger } from '@aztec/foundation/log'; -import { createAztecNodeClient } from '@aztec/types'; import { getPXEServiceConfig } from '../config/index.js'; import { startPXEHttpServer } from '../pxe_http/index.js'; diff --git a/yarn-project/pxe/tsconfig.json b/yarn-project/pxe/tsconfig.json index 907a410c56b..cf03aac99d4 100644 --- a/yarn-project/pxe/tsconfig.json +++ b/yarn-project/pxe/tsconfig.json @@ -9,6 +9,9 @@ { "path": "../acir-simulator" }, + { + "path": "../aztec.js" + }, { "path": "../circuits.js" }, diff --git a/yarn-project/types/src/aztec_node/rpc/index.ts b/yarn-project/types/src/aztec_node/rpc/index.ts deleted file mode 100644 index 9b73b528764..00000000000 --- a/yarn-project/types/src/aztec_node/rpc/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './aztec_node_client.js'; diff --git a/yarn-project/types/src/contract_data.ts b/yarn-project/types/src/contract_data.ts index 093a14b0176..6372024f782 100644 --- a/yarn-project/types/src/contract_data.ts +++ b/yarn-project/types/src/contract_data.ts @@ -103,6 +103,23 @@ export class EncodedContractFunction { return new EncodedContractFunction(fnSelector, isInternal, reader.readBuffer()); } + /** + * Serializes this instance into a string. + * @returns Encoded string. + */ + toString(): string { + return this.toBuffer().toString('hex'); + } + + /** + * Deserializes a contract function object from an encoded string. + * @param data - The encoded string. + * @returns The deserialized contract function. + */ + static fromString(data: string): EncodedContractFunction { + return EncodedContractFunction.fromBuffer(Buffer.from(data, 'hex')); + } + /** * Creates a random contract function. * @returns A random contract function. diff --git a/yarn-project/types/src/index.ts b/yarn-project/types/src/index.ts index 149ec5a7a6a..f97534f66da 100644 --- a/yarn-project/types/src/index.ts +++ b/yarn-project/types/src/index.ts @@ -1,3 +1,8 @@ +import { ContractDataSource } from './contract_data.js'; +import { L2LogsSource } from './index.js'; +import { L1ToL2MessageSource } from './l1_to_l2_message.js'; +import { L2BlockSource } from './l2_block_source.js'; + export * from './constants.js'; export * from './contract_dao.js'; export * from './contract_database.js'; @@ -22,5 +27,8 @@ export * from './packed_arguments.js'; export * from './interfaces/index.js'; export * from './sibling_path.js'; export * from './auth_witness.js'; -export * from './aztec_node/rpc/index.js'; + export { CompleteAddress, PublicKey, PartialAddress, GrumpkinPrivateKey } from '@aztec/circuits.js'; + +/** Helper type for archival data sources. */ +export type ArchiveSource = L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource; diff --git a/yarn-project/types/src/l1_to_l2_message.ts b/yarn-project/types/src/l1_to_l2_message.ts index fffc5ca64c3..b962bf5642a 100644 --- a/yarn-project/types/src/l1_to_l2_message.ts +++ b/yarn-project/types/src/l1_to_l2_message.ts @@ -155,6 +155,15 @@ export class L1ToL2Message { return new L1ToL2Message(sender, recipient, content, secretHash, deadline, fee); } + toString(): string { + return this.toBuffer().toString('hex'); + } + + static fromString(data: string): L1ToL2Message { + const buffer = Buffer.from(data, 'hex'); + return L1ToL2Message.fromBuffer(buffer); + } + static empty(): L1ToL2Message { return new L1ToL2Message(L1Actor.empty(), L2Actor.empty(), Fr.ZERO, Fr.ZERO, 0, 0); } diff --git a/yarn-project/types/src/logs/l2_block_l2_logs.ts b/yarn-project/types/src/logs/l2_block_l2_logs.ts index a0c095c3aa6..3b298f06a41 100644 --- a/yarn-project/types/src/logs/l2_block_l2_logs.ts +++ b/yarn-project/types/src/logs/l2_block_l2_logs.ts @@ -56,6 +56,24 @@ export class L2BlockL2Logs { return new L2BlockL2Logs(txLogs); } + /** + * Seralizes logs into a string. + * @returns A string representation of the serialized logs. + */ + public toString(): string { + return this.toBuffer().toString('hex'); + } + + /** + * Deserializes logs from a string. + * @param data - The string containing the serialized logs. + * @returns A new `L2BlockL2Logs` object. + */ + public static fromString(data: string): L2BlockL2Logs { + const buffer = Buffer.from(data, 'hex'); + return L2BlockL2Logs.fromBuffer(buffer); + } + /** * Creates a new `L2BlockL2Logs` object with `numCalls` function logs and `numLogsPerCall` logs in each function * call. diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index fc56b14cff8..f5500fb5520 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -143,6 +143,7 @@ __metadata: resolution: "@aztec/aztec-node@workspace:aztec-node" dependencies: "@aztec/archiver": "workspace:^" + "@aztec/aztec.js": "workspace:^" "@aztec/circuits.js": "workspace:^" "@aztec/ethereum": "workspace:^" "@aztec/foundation": "workspace:^" @@ -193,9 +194,11 @@ __metadata: "@types/jest": ^29.5.0 "@types/koa": ^2.13.6 abitype: ^0.8.11 + commander: ^11.1.0 jest: ^29.5.0 koa: ^2.14.2 koa-router: ^12.0.0 + lodash.pick: ^4.4.0 ts-jest: ^29.1.0 ts-node: ^10.9.1 typescript: ^5.0.4 @@ -701,6 +704,7 @@ __metadata: resolution: "@aztec/pxe@workspace:pxe" dependencies: "@aztec/acir-simulator": "workspace:^" + "@aztec/aztec.js": "workspace:^" "@aztec/circuits.js": "workspace:^" "@aztec/ethereum": "workspace:^" "@aztec/foundation": "workspace:^" @@ -5444,6 +5448,13 @@ __metadata: languageName: node linkType: hard +"commander@npm:^11.1.0": + version: 11.1.0 + resolution: "commander@npm:11.1.0" + checksum: fd1a8557c6b5b622c89ecdfde703242ab7db3b628ea5d1755784c79b8e7cb0d74d65b4a262289b533359cd58e1bfc0bf50245dfbcd2954682a6f367c828b79ef + languageName: node + linkType: hard + "commander@npm:^2.16.0, commander@npm:^2.20.0, commander@npm:^2.20.3, commander@npm:^2.8.1": version: 2.20.3 resolution: "commander@npm:2.20.3" From 47f2bcf3a2ebd09641a71b3951303954c6f546f0 Mon Sep 17 00:00:00 2001 From: spypsy Date: Mon, 8 Jan 2024 15:49:05 +0000 Subject: [PATCH 02/50] finish renaming --- boxes/Dockerfile | 4 +- boxes/docker-compose.yml | 2 +- docs/docs/dev_docs/cli/cli-commands.md | 9 +- docs/docs/dev_docs/contracts/deploying.md | 2 +- docs/docs/dev_docs/updating.md | 8 +- yarn-project/Dockerfile.prod | 64 +++++------ yarn-project/aztec-node/terraform/main.tf | 4 +- yarn-project/aztec/Dockerfile | 2 +- yarn-project/aztec/README.md | 8 +- yarn-project/aztec/docker-compose.yml | 6 +- yarn-project/aztec/package.json | 2 +- yarn-project/aztec/src/logging.ts | 4 +- yarn-project/deploy_npm.sh | 2 +- yarn-project/end-to-end/package.json | 2 +- .../scripts/docker-compose-browser.yml | 2 +- .../end-to-end/scripts/docker-compose-p2p.yml | 2 +- .../end-to-end/scripts/docker-compose.yml | 2 +- .../src/e2e_aztec_js_browser.test.ts | 4 +- yarn-project/end-to-end/src/e2e_cli.test.ts | 2 +- .../src/guides/dapp_testing.test.ts | 2 +- .../end-to-end/src/sample-dapp/index.test.mjs | 2 +- yarn-project/end-to-end/tsconfig.json | 2 +- .../scripts/docker-compose-bootstrap.yml | 2 +- yarn-project/package.json | 2 +- yarn-project/tsconfig.json | 108 +++++++++++++----- yarn-project/yarn.lock | 72 ++++++------ 26 files changed, 187 insertions(+), 134 deletions(-) diff --git a/boxes/Dockerfile b/boxes/Dockerfile index 963e664c5db..4191b3559a6 100644 --- a/boxes/Dockerfile +++ b/boxes/Dockerfile @@ -1,11 +1,11 @@ # Builds the boxes (they were copied into yarn-project-base so the cli can unbox). # Produces a container that can be run to test a specific box. See docker-compose.yml. -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/aztec-sandbox AS aztec-sandbox +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/aztec AS aztec FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir as noir # We need yarn. Start fresh container. FROM node:18.19.0 -COPY --from=aztec-sandbox /usr/src /usr/src +COPY --from=aztec /usr/src /usr/src COPY --from=noir /usr/src/noir/target/release/nargo /usr/src/noir/target/release/nargo WORKDIR /usr/src/boxes ENV AZTEC_NARGO=/usr/src/noir/target/release/nargo diff --git a/boxes/docker-compose.yml b/boxes/docker-compose.yml index f916d56c810..e731d25c01e 100644 --- a/boxes/docker-compose.yml +++ b/boxes/docker-compose.yml @@ -5,7 +5,7 @@ services: command: "'anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337'" aztec: - image: aztecprotocol/aztec-sandbox + image: aztecprotocol/aztec environment: ETHEREUM_HOST: http://ethereum:8545 CHAIN_ID: 31337 diff --git a/docs/docs/dev_docs/cli/cli-commands.md b/docs/docs/dev_docs/cli/cli-commands.md index 7a7c9e38761..2d89078d21e 100644 --- a/docs/docs/dev_docs/cli/cli-commands.md +++ b/docs/docs/dev_docs/cli/cli-commands.md @@ -15,20 +15,20 @@ The CLI will be installed automatically via Docker by running the command to ins :::info -The `@aztec/aztec-sandbox` and `@aztec/cli` packages published to npm **should not be used**, in favor of Docker. If you've installed the sandbox or the CLI via NPM, **uninstall** them and remove them from your project dependencies and [install via Docker](./sandbox-reference.md#with-docker). +The `@aztec/aztec` and `@aztec/cli` packages published to npm **should not be used**, in favor of Docker. If you've installed the sandbox or the CLI via NPM, **uninstall** them and remove them from your project dependencies and [install via Docker](./sandbox-reference.md#with-docker). - + -yarn global remove @aztec/aztec-sandbox @aztec/cli +yarn global remove @aztec/aztec @aztec/cli -npm -g uninstall @aztec/aztec-sandbox @aztec/cli +npm -g uninstall @aztec/aztec @aztec/cli @@ -49,7 +49,6 @@ This command does a few things to manage updates: - It looks for `Nargo.toml` at the `--contract` paths specified and updates all `aztec.nr` dependencies to the versions the sandbox expects. - It outputs the changes. - :::info The update command won't update the CLI itself. To update these follow the [updating instructions which point to our curl command](./sandbox-reference.md#with-docker). diff --git a/docs/docs/dev_docs/contracts/deploying.md b/docs/docs/dev_docs/contracts/deploying.md index b3b25d27773..e92d5d617ef 100644 --- a/docs/docs/dev_docs/contracts/deploying.md +++ b/docs/docs/dev_docs/contracts/deploying.md @@ -6,7 +6,7 @@ Once you have [compiled](./compiling.md) your contracts you can proceed to deplo - `aztec-cli` installed (go to [CLI main section](../cli/main.md) for installation instructions) - contract artifacts ready (go to [Compiling contracts section](./compiling.md) for instructions on how to compile contracts) -- aztec-sandbox running (go to [Sandbox section](../getting_started/quickstart.md) for instructions on how to install and run the sandbox) +- Aztec Sandbox running (go to [Sandbox section](../getting_started/quickstart.md) for instructions on how to install and run the sandbox) ## Deploy diff --git a/docs/docs/dev_docs/updating.md b/docs/docs/dev_docs/updating.md index 05c9b76de53..7d96e67feef 100644 --- a/docs/docs/dev_docs/updating.md +++ b/docs/docs/dev_docs/updating.md @@ -3,20 +3,20 @@ title: Updating --- :::info -The `@aztec/aztec-sandbox` and `@aztec/cli` packages published to npm **should not be used**, in favor of Docker. If you've installed the sandbox or the CLI via NPM, **uninstall** them and remove them from your project dependencies and [install via Docker](./cli/sandbox-reference.md#with-docker). +The `@aztec/aztec` and `@aztec/cli` packages published to npm **should not be used**, in favor of Docker. If you've installed the sandbox or the CLI via NPM, **uninstall** them and remove them from your project dependencies and [install via Docker](./cli/sandbox-reference.md#with-docker). - + -yarn global remove @aztec/aztec-sandbox @aztec/cli +yarn global remove @aztec/aztec @aztec/cli -npm -g uninstall @aztec/aztec-sandbox @aztec/cli +npm -g uninstall @aztec/aztec @aztec/cli diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod index 8b76b9c2323..680fb585d72 100644 --- a/yarn-project/Dockerfile.prod +++ b/yarn-project/Dockerfile.prod @@ -11,13 +11,13 @@ ARG COMMIT_TAG="" # TODO: Use release-please to update package.json directly, and remove this! RUN ./scripts/version_packages.sh # Productionify. See comment in yarn-project-base/Dockerfile. -RUN yarn workspaces focus @aztec/cli @aztec/aztec-sandbox @aztec/aztec-faucet @aztec/aztec.js --production && \ - yarn cache clean && \ - rm -rf /usr/src/barretenberg/ts/src && \ - # TODO: Fix by extracting noir code out of yarn-project. - # This was just a "rm -rf ./**/src". - # Due to the mess of us needing to find noir code in noir-protocol-circuits/src/crates we have to do this... - find . -maxdepth 2 -name src -type d | grep -v "./noir-protocol-circuits" | xargs rm -rf +RUN yarn workspaces focus @aztec/cli @aztec/aztec @aztec/aztec-faucet @aztec/aztec.js --production && \ + yarn cache clean && \ + rm -rf /usr/src/barretenberg/ts/src && \ + # TODO: Fix by extracting noir code out of yarn-project. + # This was just a "rm -rf ./**/src". + # Due to the mess of us needing to find noir code in noir-protocol-circuits/src/crates we have to do this... + find . -maxdepth 2 -name src -type d | grep -v "./noir-protocol-circuits" | xargs rm -rf # We no longer need nargo. RUN rm -rf /usr/src/noir/target @@ -32,32 +32,32 @@ FROM ubuntu:lunar RUN apt update && apt install -y curl && rm -rf /var/lib/apt/lists/* && apt-get clean ENV NODE_VERSION=18.19.0 RUN ARCH= && \ - dpkgArch="$(dpkg --print-architecture)" && \ - case "${dpkgArch##*-}" in \ - amd64) ARCH='x64';; \ - arm64) ARCH='arm64';; \ - *) echo "unsupported architecture"; exit 1 ;; \ - esac && \ - curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.gz" && \ - tar zxf "node-v$NODE_VERSION-linux-$ARCH.tar.gz" -C /usr --strip-components=1 --no-same-owner \ - --exclude "*/share/*" \ - --exclude "*/bin/corepack" \ - --exclude "*/bin/npx" \ - --exclude "*/bin/npm" \ - --exclude "*/corepack/*" \ - --exclude "*/npm/man/*" \ - --exclude "*/npm/docs/*" \ - --exclude "*/include/*" && \ - rm "node-v$NODE_VERSION-linux-$ARCH.tar.gz" && \ - node --version + dpkgArch="$(dpkg --print-architecture)" && \ + case "${dpkgArch##*-}" in \ + amd64) ARCH='x64';; \ + arm64) ARCH='arm64';; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac && \ + curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.gz" && \ + tar zxf "node-v$NODE_VERSION-linux-$ARCH.tar.gz" -C /usr --strip-components=1 --no-same-owner \ + --exclude "*/share/*" \ + --exclude "*/bin/corepack" \ + --exclude "*/bin/npx" \ + --exclude "*/bin/npm" \ + --exclude "*/corepack/*" \ + --exclude "*/npm/man/*" \ + --exclude "*/npm/docs/*" \ + --exclude "*/include/*" && \ + rm "node-v$NODE_VERSION-linux-$ARCH.tar.gz" && \ + node --version # Yarn is used for unboxing. -ENV YARN_VERSION=1.22.19 +ENV YARN_VERSION=1.22.19 RUN curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" && \ - mkdir -p /opt && \ - tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ && \ - ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn && \ - ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg && \ - rm yarn-v$YARN_VERSION.tar.gz && \ - yarn --version + mkdir -p /opt && \ + tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ && \ + ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn && \ + ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg && \ + rm yarn-v$YARN_VERSION.tar.gz && \ + yarn --version COPY --from=builder /usr/src /usr/src ENTRYPOINT ["/usr/bin/node"] diff --git a/yarn-project/aztec-node/terraform/main.tf b/yarn-project/aztec-node/terraform/main.tf index becb03ad1b5..a65230f6a16 100644 --- a/yarn-project/aztec-node/terraform/main.tf +++ b/yarn-project/aztec-node/terraform/main.tf @@ -62,7 +62,7 @@ locals { "/dns4/${var.DEPLOY_TAG}-p2p-bootstrap-${i + 1}.local/tcp/${var.BOOTNODE_LISTEN_PORT + i}/p2p/${local.bootnode_ids[i]}" ] combined_bootnodes = join(",", local.bootnodes) - data_dir = "/usr/src/yarn-project/aztec-sandbox/data" + data_dir = "/usr/src/yarn-project/aztec/data" } resource "aws_cloudwatch_log_group" "aztec-node-log-group" { @@ -154,7 +154,7 @@ resource "aws_ecs_task_definition" "aztec-node" { [ { "name": "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}", - "image": "${var.DOCKERHUB_ACCOUNT}/aztec-sandbox:${var.DEPLOY_TAG}", + "image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}", "essential": true, "memoryReservation": 3776, "portMappings": [ diff --git a/yarn-project/aztec/Dockerfile b/yarn-project/aztec/Dockerfile index afdfb2094e1..d36274c527c 100644 --- a/yarn-project/aztec/Dockerfile +++ b/yarn-project/aztec/Dockerfile @@ -1,5 +1,5 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod -ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/aztec-sandbox/dest/bin/index.js"] +ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/aztec/dest/bin/index.js"] EXPOSE 8079 8080 # The version has been updated in yarn-project-prod. diff --git a/yarn-project/aztec/README.md b/yarn-project/aztec/README.md index b901d0b8b9d..2c01123c7b4 100644 --- a/yarn-project/aztec/README.md +++ b/yarn-project/aztec/README.md @@ -1,6 +1,6 @@ -# aztec-sandbox +# aztec -Aztec Sandbox is a package that allows for a simple development environment on Aztec stack. It creates a Private eXecution Environment (PXE) that listens for HTTP requests on `localhost:8080` by default. When started, it deploys all necessary L1 Aztec contracts and then starts listening for RPC requests. +Aztec is a package that allows for a simple development environment on Aztec stack. It creates a Private eXecution Environment (PXE) that listens for HTTP requests on `localhost:8080` by default. When started, it deploys all necessary L1 Aztec contracts and then starts listening for RPC requests. ## How to run: @@ -38,7 +38,7 @@ Before running locally you'll need to: - `yarn build` And you should be good to go! -From the `aztec-sandbox` directory, you can run the two existing examples: +From the `aztec` directory, you can run the two existing examples: - Deployment, mint and transfer on an Aztec Private Token - `yarn run:example:token` @@ -54,4 +54,4 @@ export FORK_URL= ## Publishing This package is set-up to be published on dockerhub by CI whenever there's a tagged release on `master` branch. -It's published under the tags `aztecprotocol/aztec-sandbox:latest` & `aztecprotocol/aztec-sandbox:`. +It's published under the tags `aztecprotocol/aztec:latest` & `aztecprotocol/aztec:`. diff --git a/yarn-project/aztec/docker-compose.yml b/yarn-project/aztec/docker-compose.yml index 2741f74c069..04ec4e7d9bc 100644 --- a/yarn-project/aztec/docker-compose.yml +++ b/yarn-project/aztec/docker-compose.yml @@ -7,7 +7,7 @@ services: if [ -n "$FORK_BLOCK_NUMBER" ] && [ -n "$FORK_URL" ]; then exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" elif [ -n "$FORK_URL" ]; then - exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent --fork-url "$FORK_URL" + exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent --fork-url "$FORK_URL" else exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent fi' @@ -15,7 +15,7 @@ services: - '${SANDBOX_ANVIL_PORT:-8545}:8545' aztec: - image: 'aztecprotocol/aztec-sandbox:${SANDBOX_VERSION:-latest}' + image: 'aztecprotocol/aztec:${SANDBOX_VERSION:-latest}' ports: - '${SANDBOX_AZTEC_NODE_PORT:-8079}:8079' - '${SANDBOX_PXE_PORT:-8080}:8080' @@ -31,4 +31,4 @@ services: PXE_BLOCK_POLLING_INTERVAL_MS: 50 ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 volumes: - - ./log:/usr/src/yarn-project/aztec-sandbox/log:rw + - ./log:/usr/src/yarn-project/aztec/log:rw diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index 2142886e020..91d245d744a 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -74,4 +74,4 @@ "engines": { "node": ">=18" } -} \ No newline at end of file +} diff --git a/yarn-project/aztec/src/logging.ts b/yarn-project/aztec/src/logging.ts index a9de4c40415..fe184716efc 100644 --- a/yarn-project/aztec/src/logging.ts +++ b/yarn-project/aztec/src/logging.ts @@ -6,14 +6,14 @@ import * as winston from 'winston'; import DailyRotateFile from 'winston-daily-rotate-file'; const { format } = winston; -const CURRENT_LOG_FILE_NAME = 'aztec-sandbox.debug.log'; +const CURRENT_LOG_FILE_NAME = 'aztec.debug.log'; const LOG_DIR = 'log'; /** Creates a winston logger that logs everything to a local rotating file */ function createWinstonLogger() { // See https://www.npmjs.com/package/winston-daily-rotate-file#user-content-options const transport: DailyRotateFile = new DailyRotateFile({ - filename: 'aztec-sandbox-%DATE%.debug.log', + filename: 'aztec-%DATE%.debug.log', dirname: LOG_DIR, datePattern: 'YYYY-MM-DD', zippedArchive: true, diff --git a/yarn-project/deploy_npm.sh b/yarn-project/deploy_npm.sh index 19c58b80d6c..c9f8a651cc4 100755 --- a/yarn-project/deploy_npm.sh +++ b/yarn-project/deploy_npm.sh @@ -98,4 +98,4 @@ deploy_package p2p deploy_package world-state deploy_package sequencer-client deploy_package aztec-node -deploy_package aztec-sandbox +deploy_package aztec diff --git a/yarn-project/end-to-end/package.json b/yarn-project/end-to-end/package.json index d3059c62120..1fc8979efe4 100644 --- a/yarn-project/end-to-end/package.json +++ b/yarn-project/end-to-end/package.json @@ -23,8 +23,8 @@ }, "dependencies": { "@aztec/archiver": "workspace:^", + "@aztec/aztec": "workspace:^", "@aztec/aztec-node": "workspace:^", - "@aztec/aztec-sandbox": "workspace:^", "@aztec/aztec.js": "workspace:^", "@aztec/circuits.js": "workspace:^", "@aztec/cli": "workspace:^", diff --git a/yarn-project/end-to-end/scripts/docker-compose-browser.yml b/yarn-project/end-to-end/scripts/docker-compose-browser.yml index d3abd8fb74d..f859de7de39 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-browser.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-browser.yml @@ -13,7 +13,7 @@ services: - '8545:8545' sandbox: - image: aztecprotocol/aztec-sandbox:latest + image: aztecprotocol/aztec:latest environment: DEBUG: 'aztec:*' DEBUG_COLORS: 1 diff --git a/yarn-project/end-to-end/scripts/docker-compose-p2p.yml b/yarn-project/end-to-end/scripts/docker-compose-p2p.yml index 3bca942dd23..4088526269f 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-p2p.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-p2p.yml @@ -7,7 +7,7 @@ services: - '8545:8545' p2p-bootstrap: - image: aztecprotocol/aztec-sandbox:latest + image: aztecprotocol/aztec:latest ports: - '40400:40400' command: 'start' diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 9b675470f35..6a05e652777 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -13,7 +13,7 @@ services: - '8545:8545' sandbox: - image: aztecprotocol/aztec-sandbox:latest + image: aztecprotocol/aztec:latest environment: DEBUG: 'aztec:*' DEBUG_COLORS: 1 diff --git a/yarn-project/end-to-end/src/e2e_aztec_js_browser.test.ts b/yarn-project/end-to-end/src/e2e_aztec_js_browser.test.ts index b34bdb5334b..e1359f20060 100644 --- a/yarn-project/end-to-end/src/e2e_aztec_js_browser.test.ts +++ b/yarn-project/end-to-end/src/e2e_aztec_js_browser.test.ts @@ -19,11 +19,11 @@ const pageLogger = createDebugLogger('aztec:canary_aztec.js:web:page'); * 2) go to `yarn-project/aztec.js` and build the web packed package with `yarn build:web`, * 3) start anvil: `anvil`, * 4) open new terminal and optionally set the more verbose debug level: `DEBUG=aztec:*`, - * 5) go to the sandbox dir `yarn-project/aztec-sandbox` and run `yarn start`, + * 5) go to the sandbox dir `yarn-project/aztec` and run `yarn start`, * 6) open new terminal and export the URL of PXE from Sandbox: `export PXE_URL='http://localhost:8080'`, * 7) go to `yarn-project/end-to-end` and run the test: `yarn test aztec_js_browser` * - * NOTE: If you see aztec-sandbox logs spammed with unexpected logs there is probably a chrome process with a webpage + * NOTE: If you see aztec sandbox logs spammed with unexpected logs there is probably a chrome process with a webpage * unexpectedly running in the background. Kill it with `killall chrome` */ const setupApp = () => { diff --git a/yarn-project/end-to-end/src/e2e_cli.test.ts b/yarn-project/end-to-end/src/e2e_cli.test.ts index bb3d0a11d71..ce7575b8fd6 100644 --- a/yarn-project/end-to-end/src/e2e_cli.test.ts +++ b/yarn-project/end-to-end/src/e2e_cli.test.ts @@ -1,4 +1,4 @@ -import { startHttpRpcServer } from '@aztec/aztec-sandbox'; +import { startHttpRpcServer } from '@aztec/aztec'; import { PXE, createDebugLogger } from '@aztec/aztec.js'; import { createPXERpcServer } from '@aztec/pxe'; diff --git a/yarn-project/end-to-end/src/guides/dapp_testing.test.ts b/yarn-project/end-to-end/src/guides/dapp_testing.test.ts index 5a5d5bda30a..880d7e9379a 100644 --- a/yarn-project/end-to-end/src/guides/dapp_testing.test.ts +++ b/yarn-project/end-to-end/src/guides/dapp_testing.test.ts @@ -1,4 +1,4 @@ -import { createSandbox } from '@aztec/aztec-sandbox'; +import { createSandbox } from '@aztec/aztec'; import { AccountWallet, CheatCodes, diff --git a/yarn-project/end-to-end/src/sample-dapp/index.test.mjs b/yarn-project/end-to-end/src/sample-dapp/index.test.mjs index 7441e6ba7e6..341eea3ddac 100644 --- a/yarn-project/end-to-end/src/sample-dapp/index.test.mjs +++ b/yarn-project/end-to-end/src/sample-dapp/index.test.mjs @@ -1,4 +1,4 @@ -import { createSandbox } from '@aztec/aztec-sandbox'; +import { createSandbox } from '@aztec/aztec'; import { Contract, ExtendedNote, diff --git a/yarn-project/end-to-end/tsconfig.json b/yarn-project/end-to-end/tsconfig.json index c44790b98d8..b3ef03eca9e 100644 --- a/yarn-project/end-to-end/tsconfig.json +++ b/yarn-project/end-to-end/tsconfig.json @@ -13,7 +13,7 @@ "path": "../aztec-node" }, { - "path": "../aztec-sandbox" + "path": "../aztec" }, { "path": "../aztec.js" diff --git a/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml b/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml index ce5913b9e75..236b889dab6 100644 --- a/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml +++ b/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml @@ -1,7 +1,7 @@ version: '3' services: p2p-bootstrap: - image: 278380418400.dkr.ecr.eu-west-2.amazonaws.com/aztec-sandbox:latest + image: 278380418400.dkr.ecr.eu-west-2.amazonaws.com/aztec:latest ports: - '40400:40400' environment: diff --git a/yarn-project/package.json b/yarn-project/package.json index a387fc315b8..4a9a114bc73 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -69,4 +69,4 @@ "@noir-lang/noir_wasm": "portal:../noir/packages/noir_wasm", "@noir-lang/noir_js": "portal:../noir/packages/noir_js" } -} \ No newline at end of file +} diff --git a/yarn-project/tsconfig.json b/yarn-project/tsconfig.json index a57263c1824..aaddf91b46b 100644 --- a/yarn-project/tsconfig.json +++ b/yarn-project/tsconfig.json @@ -1,7 +1,11 @@ { "compilerOptions": { "target": "es2020", - "lib": ["dom", "esnext", "es2017.object"], + "lib": [ + "dom", + "esnext", + "es2017.object" + ], "module": "NodeNext", "moduleResolution": "NodeNext", "strict": true, @@ -18,30 +22,80 @@ "skipLibCheck": true }, "references": [ - { "path": "acir-simulator/tsconfig.json" }, - { "path": "archiver/tsconfig.json" }, - { "path": "aztec-faucet/tsconfig.json" }, - { "path": "aztec.js/tsconfig.json" }, - { "path": "aztec-node/tsconfig.json" }, - { "path": "pxe/tsconfig.json" }, - { "path": "aztec-sandbox/tsconfig.json" }, - { "path": "circuits.js/tsconfig.json" }, - { "path": "cli/tsconfig.json" }, - { "path": "end-to-end/tsconfig.json" }, - { "path": "foundation/tsconfig.json" }, - { "path": "key-store/tsconfig.json" }, - { "path": "l1-artifacts/tsconfig.json" }, - { "path": "merkle-tree/tsconfig.json" }, - { "path": "noir-contracts/tsconfig.json" }, - { "path": "noir-compiler/tsconfig.json" }, - { "path": "noir-protocol-circuits/tsconfig.json" }, - { "path": "p2p/tsconfig.json" }, - { "path": "p2p-bootstrap/tsconfig.json" }, - { "path": "prover-client/tsconfig.json" }, - { "path": "sequencer-client/tsconfig.json" }, - { "path": "types/tsconfig.json" }, - { "path": "world-state/tsconfig.json" }, - { "path": "scripts/tsconfig.json" } + { + "path": "acir-simulator/tsconfig.json" + }, + { + "path": "archiver/tsconfig.json" + }, + { + "path": "aztec-faucet/tsconfig.json" + }, + { + "path": "aztec.js/tsconfig.json" + }, + { + "path": "aztec-node/tsconfig.json" + }, + { + "path": "pxe/tsconfig.json" + }, + { + "path": "aztec/tsconfig.json" + }, + { + "path": "circuits.js/tsconfig.json" + }, + { + "path": "cli/tsconfig.json" + }, + { + "path": "end-to-end/tsconfig.json" + }, + { + "path": "foundation/tsconfig.json" + }, + { + "path": "key-store/tsconfig.json" + }, + { + "path": "l1-artifacts/tsconfig.json" + }, + { + "path": "merkle-tree/tsconfig.json" + }, + { + "path": "noir-contracts/tsconfig.json" + }, + { + "path": "noir-compiler/tsconfig.json" + }, + { + "path": "noir-protocol-circuits/tsconfig.json" + }, + { + "path": "p2p/tsconfig.json" + }, + { + "path": "p2p-bootstrap/tsconfig.json" + }, + { + "path": "prover-client/tsconfig.json" + }, + { + "path": "sequencer-client/tsconfig.json" + }, + { + "path": "types/tsconfig.json" + }, + { + "path": "world-state/tsconfig.json" + }, + { + "path": "scripts/tsconfig.json" + } ], - "files": ["./@types/jest/index.d.ts"] -} + "files": [ + "./@types/jest/index.d.ts" + ] +} \ No newline at end of file diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index f5500fb5520..a8d79867f10 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -175,41 +175,6 @@ __metadata: languageName: unknown linkType: soft -"@aztec/aztec-sandbox@workspace:^, @aztec/aztec-sandbox@workspace:aztec-sandbox": - version: 0.0.0-use.local - resolution: "@aztec/aztec-sandbox@workspace:aztec-sandbox" - dependencies: - "@aztec/aztec-node": "workspace:^" - "@aztec/aztec.js": "workspace:^" - "@aztec/circuits.js": "workspace:^" - "@aztec/ethereum": "workspace:^" - "@aztec/foundation": "workspace:^" - "@aztec/l1-artifacts": "workspace:^" - "@aztec/noir-compiler": "workspace:^" - "@aztec/noir-contracts": "workspace:^" - "@aztec/p2p": "workspace:^" - "@aztec/pxe": "workspace:^" - "@aztec/types": "workspace:^" - "@jest/globals": ^29.5.0 - "@types/jest": ^29.5.0 - "@types/koa": ^2.13.6 - abitype: ^0.8.11 - commander: ^11.1.0 - jest: ^29.5.0 - koa: ^2.14.2 - koa-router: ^12.0.0 - lodash.pick: ^4.4.0 - ts-jest: ^29.1.0 - ts-node: ^10.9.1 - typescript: ^5.0.4 - viem: ^1.2.5 - winston: ^3.10.0 - winston-daily-rotate-file: ^4.7.1 - bin: - aztec-sandbox: ./dest/bin/index.js - languageName: unknown - linkType: soft - "@aztec/aztec.js@workspace:^, @aztec/aztec.js@workspace:aztec.js": version: 0.0.0-use.local resolution: "@aztec/aztec.js@workspace:aztec.js" @@ -260,6 +225,41 @@ __metadata: languageName: unknown linkType: soft +"@aztec/aztec@workspace:^, @aztec/aztec@workspace:aztec": + version: 0.0.0-use.local + resolution: "@aztec/aztec@workspace:aztec" + dependencies: + "@aztec/aztec-node": "workspace:^" + "@aztec/aztec.js": "workspace:^" + "@aztec/circuits.js": "workspace:^" + "@aztec/ethereum": "workspace:^" + "@aztec/foundation": "workspace:^" + "@aztec/l1-artifacts": "workspace:^" + "@aztec/noir-compiler": "workspace:^" + "@aztec/noir-contracts": "workspace:^" + "@aztec/p2p": "workspace:^" + "@aztec/pxe": "workspace:^" + "@aztec/types": "workspace:^" + "@jest/globals": ^29.5.0 + "@types/jest": ^29.5.0 + "@types/koa": ^2.13.6 + abitype: ^0.8.11 + commander: ^11.1.0 + jest: ^29.5.0 + koa: ^2.14.2 + koa-router: ^12.0.0 + lodash.pick: ^4.4.0 + ts-jest: ^29.1.0 + ts-node: ^10.9.1 + typescript: ^5.0.4 + viem: ^1.2.5 + winston: ^3.10.0 + winston-daily-rotate-file: ^4.7.1 + bin: + aztec: ./dest/bin/index.js + languageName: unknown + linkType: soft + "@aztec/bb.js@portal:../barretenberg/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A.": version: 0.0.0-use.local resolution: "@aztec/bb.js@portal:../barretenberg/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A." @@ -347,8 +347,8 @@ __metadata: resolution: "@aztec/end-to-end@workspace:end-to-end" dependencies: "@aztec/archiver": "workspace:^" + "@aztec/aztec": "workspace:^" "@aztec/aztec-node": "workspace:^" - "@aztec/aztec-sandbox": "workspace:^" "@aztec/aztec.js": "workspace:^" "@aztec/circuits.js": "workspace:^" "@aztec/cli": "workspace:^" From 89ec92c3604ebe6583adf304603b7bba5ce6cc72 Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 9 Jan 2024 10:33:52 +0000 Subject: [PATCH 03/50] add standalone archiver --- aztec-up/bin/docker-compose.yml | 4 +- yarn-project/archiver/src/archiver/index.ts | 1 + .../aztec-node/src/aztec-node/server.ts | 3 +- yarn-project/aztec/package.json | 2 + yarn-project/aztec/src/bin/cli.ts | 122 +++++++++++++----- yarn-project/aztec/src/sandbox.ts | 4 +- yarn-project/aztec/tsconfig.json | 6 + yarn-project/end-to-end/tsconfig.json | 4 +- yarn-project/kv-store/package.json | 5 +- yarn-project/kv-store/src/index.ts | 1 + .../db.ts => kv-store/src/open_db.ts} | 13 +- yarn-project/typedoc.json | 4 +- yarn-project/yarn.lock | 5 + 13 files changed, 125 insertions(+), 49 deletions(-) rename yarn-project/{aztec-node/src/aztec-node/db.ts => kv-store/src/open_db.ts} (89%) diff --git a/aztec-up/bin/docker-compose.yml b/aztec-up/bin/docker-compose.yml index a1d1e3e646c..d99d9d27cf1 100644 --- a/aztec-up/bin/docker-compose.yml +++ b/aztec-up/bin/docker-compose.yml @@ -17,7 +17,7 @@ services: ANVIL_PORT: ${ANVIL_PORT:-8545} aztec: - image: "aztecprotocol/aztec-sandbox" + image: "aztecprotocol/aztec" ports: - "${AZTEC_NODE_PORT:-8079}:${AZTEC_NODE_PORT:-8079}" - "${PXE_PORT:-8080}:${PXE_PORT:-8080}" @@ -35,4 +35,4 @@ services: AZTEC_NODE_PORT: ${AZTEC_NODE_PORT:-8079} PXE_PORT: ${PXE_PORT:-8080} volumes: - - ./log:/usr/src/yarn-project/aztec-sandbox/log:rw + - ./log:/usr/src/yarn-project/aztec/log:rw diff --git a/yarn-project/archiver/src/archiver/index.ts b/yarn-project/archiver/src/archiver/index.ts index 0ef2b0025c8..036e32f5a68 100644 --- a/yarn-project/archiver/src/archiver/index.ts +++ b/yarn-project/archiver/src/archiver/index.ts @@ -3,3 +3,4 @@ export * from './config.js'; export { MemoryArchiverStore } from './memory_archiver_store/memory_archiver_store.js'; export { LMDBArchiverStore } from './lmdb_archiver_store.js'; export { ArchiverDataStore } from './archiver_store.js'; +export * from './archiver_http_server.js'; diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 8f3777e51ef..387b9de1b61 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -17,7 +17,7 @@ import { computeGlobalsHash, computePublicDataTreeLeafSlot } from '@aztec/circui import { L1ContractAddresses, createEthereumChain } from '@aztec/ethereum'; import { AztecAddress } from '@aztec/foundation/aztec-address'; import { createDebugLogger } from '@aztec/foundation/log'; -import { AztecLmdbStore } from '@aztec/kv-store'; +import { AztecLmdbStore, openDb } from '@aztec/kv-store'; import { AztecKVTxPool, P2P, createP2PClient } from '@aztec/p2p'; import { GlobalVariableBuilder, @@ -61,7 +61,6 @@ import { import { LevelUp } from 'levelup'; import { AztecNodeConfig } from './config.js'; -import { openDb } from './db.js'; /** * The aztec node. diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index 91d245d744a..5b8e5e27896 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -28,11 +28,13 @@ "../package.common.json" ], "dependencies": { + "@aztec/archiver": "workspace:^", "@aztec/aztec-node": "workspace:^", "@aztec/aztec.js": "workspace:^", "@aztec/circuits.js": "workspace:^", "@aztec/ethereum": "workspace:^", "@aztec/foundation": "workspace:^", + "@aztec/kv-store": "workspace:^", "@aztec/l1-artifacts": "workspace:^", "@aztec/noir-compiler": "workspace:^", "@aztec/noir-contracts": "workspace:^", diff --git a/yarn-project/aztec/src/bin/cli.ts b/yarn-project/aztec/src/bin/cli.ts index 90bd71def92..f476f02237f 100644 --- a/yarn-project/aztec/src/bin/cli.ts +++ b/yarn-project/aztec/src/bin/cli.ts @@ -1,17 +1,21 @@ -import { createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; -import { fileURLToPath } from '@aztec/aztec.js'; -import { DebugLogger, LogFn, createConsoleLogger, createDebugLogger } from '@aztec/foundation/log'; -import { createPXERpcServer, getPXEServiceConfig } from '@aztec/pxe'; +import { Archiver, LMDBArchiverStore, createArchiverRpcServer, getConfigEnvVars } from '@aztec/archiver'; +import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; +import { createAztecNodeClient, fileURLToPath } from '@aztec/aztec.js'; +import { NULL_KEY } from '@aztec/ethereum'; +import { LogFn, createConsoleLogger, createDebugLogger } from '@aztec/foundation/log'; +import { openDb } from '@aztec/kv-store'; +import { createPXERpcServer, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import { Command } from 'commander'; import { readFileSync } from 'fs'; import pick from 'lodash.pick'; import { dirname, resolve } from 'path'; +import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; -import { MNEMONIC, createAztecNode, createAztecPXE, createSandbox } from '../sandbox.js'; +import { MNEMONIC, createAztecNode, createAztecPXE, createSandbox, deployContractsToL1 } from '../sandbox.js'; import { startHttpRpcServer } from '../server.js'; -const { AZTEC_NODE_PORT = '80', PXE_PORT = '79' } = process.env; +const { AZTEC_NODE_PORT = '80', PXE_PORT = '79', ARCHIVER_PORT = '80' } = process.env; const debugLogger = createDebugLogger('aztec:cli'); @@ -47,7 +51,7 @@ const parseModuleOptions = (options: string): Record => { * @param userLog - log function for logging user output. * @param debugLogger - logger for logging debug messages. */ -export function getProgram(userLog: LogFn /*debugLogger: DebugLogger*/): Command { +export function getProgram(userLog: LogFn): Command { const program = new Command(); const packageJsonPath = resolve(dirname(fileURLToPath(import.meta.url)), '../../package.json'); @@ -91,47 +95,95 @@ export function getProgram(userLog: LogFn /*debugLogger: DebugLogger*/): Command // get config from options const nodeCliOptions = parseModuleOptions(options.node); // merge env vars and cli options - const nodeConfig = pick({ ...aztecNodeConfigEnvVars, ...nodeCliOptions }); + const nodeConfig = pick({ ...aztecNodeConfigEnvVars, ...nodeCliOptions }) as AztecNodeConfig; // if no publisher private key, then use MNEMONIC - if (nodeConfig.publisherPrivateKey === undefined) { - if (!options.archiver) { - // expect archiver url in node config - const archiverUrl = nodeCliOptions.archiverUrl; - if (!archiverUrl) { - throw new Error('Archiver Service URL is required to start Aztec Node without --archiver option'); - } - nodeConfig.archiverUrl = archiverUrl; + if (!options.archiver) { + // expect archiver url in node config + const archiverUrl = nodeCliOptions.archiverUrl; + if (!archiverUrl) { + throw new Error('Archiver Service URL is required to start Aztec Node without --archiver option'); } + nodeConfig.archiverUrl = archiverUrl; + } - const node = await createAztecNode(nodeConfig); - userLog(`Aztec Node JSON-RPC Server listening on port ${nodeCliOptions.port}`); - - // Create a PXE client that connects to the node - if (options.pxe) { - const pxe = await createAztecPXE(node); - signalHandlers.push(pxe.stop); - - // Start PXE JSON-RPC server - startHttpRpcServer(pxe, createPXERpcServer, options.PXE_PORT); + // Deploy contracts if needed + if (nodeCliOptions.deployContracts) { + let account; + if (nodeConfig.publisherPrivateKey === NULL_KEY) { + account = mnemonicToAccount(MNEMONIC); + } else { + account = privateKeyToAccount(nodeConfig.publisherPrivateKey); } + await deployContractsToL1(nodeConfig, account); + } - signalHandlers.push(node.stop); + if (!options.sequencer) { + nodeConfig.disableSequencer = true; + } else if (nodeConfig.publisherPrivateKey === NULL_KEY) { + // If we have a sequencer, ensure there's a publisher private key set. + const hdAccount = mnemonicToAccount(MNEMONIC); + const privKey = hdAccount.getHdKey().privateKey; + nodeConfig.publisherPrivateKey = `0x${Buffer.from(privKey!).toString('hex')}`; } + // P2P enabled by default + nodeConfig.p2pEnabled = nodeCliOptions.p2pDisabled !== 'true'; + + // Create and start Aztec Node. + const node = await createAztecNode(nodeConfig); + startHttpRpcServer(node, createAztecNodeRpcServer, nodeCliOptions.port || AZTEC_NODE_PORT); + userLog(`Aztec Node JSON-RPC Server listening on port ${nodeCliOptions.port || AZTEC_NODE_PORT}`); + + // Create a PXE client that connects to the node. if (options.pxe) { - // get env vars first - const pxeConfigEnvVars = getPXEServiceConfig(); - // get config from options const pxeCliOptions = parseModuleOptions(options.pxe); - // merge env vars and cli options - const pxeConfig = pick({ ...pxeConfigEnvVars, ...pxeCliOptions }); + const pxe = await createAztecPXE(node); + signalHandlers.push(pxe.stop); + + // Start PXE JSON-RPC server. + startHttpRpcServer(pxe, createPXERpcServer, pxeCliOptions.port || PXE_PORT); + } + + signalHandlers.push(node.stop); + } else if (options.pxe) { + // Starting a PXE with a remote node. + // get env vars first + const pxeConfigEnvVars = getPXEServiceConfig(); + // get config from options + const pxeCliOptions = parseModuleOptions(options.pxe); - const pxe = createPXERpcServer(pxeConfig); - // userLog(`PXE JSON-RPC Server listening on port ${pxeOptions.port}`); + // throw if no Aztec Node URL is provided + if (!pxeCliOptions.nodeUrl) { + throw new Error('Aztec Node URL (nodeUrl) option is required to start PXE without --node option'); } - installSignalHandlers(signalHandlers); + + // merge env vars and cli options + const pxeConfig = pick({ ...pxeConfigEnvVars, ...pxeCliOptions }); + + const node = createAztecNodeClient(pxeCliOptions.nodeUrl); + + const pxe = await createPXEService(node, pxeConfig); + signalHandlers.push(pxe.stop); + // Start PXE JSON-RPC server + startHttpRpcServer(pxe, createPXERpcServer, pxeCliOptions.port || PXE_PORT); + userLog(`PXE JSON-RPC Server listening on port ${pxeCliOptions.port || PXE_PORT}`); + } else if (options.archiver) { + // Start a standalone archiver. + // get env vars first + const archiverConfigEnvVars = getConfigEnvVars(); + // get config from options + const archiverCliOptions = parseModuleOptions(options.pxe); + // merge env vars and cli options + const archiverConfig = pick({ ...archiverConfigEnvVars, ...archiverCliOptions }); + + const [nodeDb] = await openDb(archiverConfig, debugLogger); + const archiverStore = new LMDBArchiverStore(nodeDb, archiverConfig.maxLogs); + + const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, true); + startHttpRpcServer(archiver, createArchiverRpcServer, archiverCliOptions.port || ARCHIVER_PORT); } + installSignalHandlers(debugLogger, signalHandlers); }); return program; } diff --git a/yarn-project/aztec/src/sandbox.ts b/yarn-project/aztec/src/sandbox.ts index d10248a6947..13b04725068 100644 --- a/yarn-project/aztec/src/sandbox.ts +++ b/yarn-project/aztec/src/sandbox.ts @@ -24,7 +24,7 @@ import { import { PXEServiceConfig, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import { AztecNode } from '@aztec/types'; -import { HDAccount, createPublicClient, http as httpViemTransport } from 'viem'; +import { HDAccount, PrivateKeyAccount, createPublicClient, http as httpViemTransport } from 'viem'; import { mnemonicToAccount } from 'viem/accounts'; import { foundry } from 'viem/chains'; @@ -72,7 +72,7 @@ async function waitThenDeploy(config: AztecNodeConfig, deployFunction: () => Pro * @param aztecNodeConfig - The Aztec Node Config * @param hdAccount - Account for publishing L1 contracts */ -export async function deployContractsToL1(aztecNodeConfig: AztecNodeConfig, hdAccount: HDAccount) { +export async function deployContractsToL1(aztecNodeConfig: AztecNodeConfig, hdAccount: HDAccount | PrivateKeyAccount) { const l1Artifacts: L1ContractArtifactsForDeployment = { contractDeploymentEmitter: { contractAbi: ContractDeploymentEmitterAbi, diff --git a/yarn-project/aztec/tsconfig.json b/yarn-project/aztec/tsconfig.json index 89a05426e77..82a1adbdcbd 100644 --- a/yarn-project/aztec/tsconfig.json +++ b/yarn-project/aztec/tsconfig.json @@ -6,6 +6,9 @@ "tsBuildInfoFile": ".tsbuildinfo" }, "references": [ + { + "path": "../archiver" + }, { "path": "../aztec-node" }, @@ -21,6 +24,9 @@ { "path": "../foundation" }, + { + "path": "../kv-store" + }, { "path": "../l1-artifacts" }, diff --git a/yarn-project/end-to-end/tsconfig.json b/yarn-project/end-to-end/tsconfig.json index b3ef03eca9e..51e0a893f55 100644 --- a/yarn-project/end-to-end/tsconfig.json +++ b/yarn-project/end-to-end/tsconfig.json @@ -10,10 +10,10 @@ "path": "../archiver" }, { - "path": "../aztec-node" + "path": "../aztec" }, { - "path": "../aztec" + "path": "../aztec-node" }, { "path": "../aztec.js" diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json index 8aea043378d..94bf287e332 100644 --- a/yarn-project/kv-store/package.json +++ b/yarn-project/kv-store/package.json @@ -26,7 +26,10 @@ }, "dependencies": { "@aztec/foundation": "workspace:^", - "lmdb": "^2.9.1" + "leveldown": "^6.1.1", + "levelup": "^5.1.1", + "lmdb": "^2.9.1", + "memdown": "^6.1.1" }, "devDependencies": { "@jest/globals": "^29.5.0", diff --git a/yarn-project/kv-store/src/index.ts b/yarn-project/kv-store/src/index.ts index 2a71333f9e6..c08d9ad3289 100644 --- a/yarn-project/kv-store/src/index.ts +++ b/yarn-project/kv-store/src/index.ts @@ -3,3 +3,4 @@ export * from './interfaces/map.js'; export * from './interfaces/singleton.js'; export * from './interfaces/store.js'; export * from './lmdb/store.js'; +export * from './open_db.js'; diff --git a/yarn-project/aztec-node/src/aztec-node/db.ts b/yarn-project/kv-store/src/open_db.ts similarity index 89% rename from yarn-project/aztec-node/src/aztec-node/db.ts rename to yarn-project/kv-store/src/open_db.ts index 9b5be428781..07894f887c5 100644 --- a/yarn-project/aztec-node/src/aztec-node/db.ts +++ b/yarn-project/kv-store/src/open_db.ts @@ -1,3 +1,4 @@ +import { EthAddress } from '@aztec/foundation/eth-address'; import { LogFn } from '@aztec/foundation/log'; import { LevelDown, default as leveldown } from 'leveldown'; @@ -7,8 +8,6 @@ import { MemDown, default as memdown } from 'memdown'; import { mkdir } from 'node:fs/promises'; import { join } from 'node:path'; -import { AztecNodeConfig } from './config.js'; - export const createMemDown = () => (memdown as any)() as MemDown; export const createLevelDown = (path: string) => (leveldown as any)(path) as LevelDown; @@ -33,7 +32,15 @@ type NodeMetadata = { * @returns The database for the aztec node. */ export async function openDb( - config: AztecNodeConfig, + config: { + /** The directory to store the database in. If not specified, a temporary database is used. */ + dataDirectory?: string; + /** The addresses of the Aztec contracts on L1. */ + l1Contracts: { + /** The address of the Aztec rollup contract on L1. */ + rollupAddress: EthAddress; + }; + }, log: LogFn, ): Promise<[nodeDb: RootDatabase, worldStateDb: LevelUp]> { const nodeMetadata: NodeMetadata = { diff --git a/yarn-project/typedoc.json b/yarn-project/typedoc.json index 408f630dc80..4ae607954f8 100644 --- a/yarn-project/typedoc.json +++ b/yarn-project/typedoc.json @@ -6,7 +6,7 @@ "archiver", "aztec-cli", "pxe", - "aztec-sandbox", + "aztec", "aztec.js", "key-store", "noir-contracts", @@ -19,4 +19,4 @@ "world-state", "merkle-tree" ] -} +} \ No newline at end of file diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index a8d79867f10..f968f6ab7d7 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -229,11 +229,13 @@ __metadata: version: 0.0.0-use.local resolution: "@aztec/aztec@workspace:aztec" dependencies: + "@aztec/archiver": "workspace:^" "@aztec/aztec-node": "workspace:^" "@aztec/aztec.js": "workspace:^" "@aztec/circuits.js": "workspace:^" "@aztec/ethereum": "workspace:^" "@aztec/foundation": "workspace:^" + "@aztec/kv-store": "workspace:^" "@aztec/l1-artifacts": "workspace:^" "@aztec/noir-compiler": "workspace:^" "@aztec/noir-contracts": "workspace:^" @@ -505,7 +507,10 @@ __metadata: "@types/node": ^18.7.23 jest: ^29.5.0 jest-mock-extended: ^3.0.3 + leveldown: ^6.1.1 + levelup: ^5.1.1 lmdb: ^2.9.1 + memdown: ^6.1.1 ts-jest: ^29.1.0 ts-node: ^10.9.1 typescript: ^5.0.4 From 8e81b1081ccdef651ee6c1f10658f1987557e5aa Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 9 Jan 2024 19:11:57 +0000 Subject: [PATCH 04/50] add p2p bootstrap node --- yarn-project/aztec/src/bin/cli.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/yarn-project/aztec/src/bin/cli.ts b/yarn-project/aztec/src/bin/cli.ts index f476f02237f..b59a24ae0ec 100644 --- a/yarn-project/aztec/src/bin/cli.ts +++ b/yarn-project/aztec/src/bin/cli.ts @@ -4,6 +4,7 @@ import { createAztecNodeClient, fileURLToPath } from '@aztec/aztec.js'; import { NULL_KEY } from '@aztec/ethereum'; import { LogFn, createConsoleLogger, createDebugLogger } from '@aztec/foundation/log'; import { openDb } from '@aztec/kv-store'; +import { BootstrapNode, getP2PConfigEnvVars } from '@aztec/p2p'; import { createPXERpcServer, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import { Command } from 'commander'; @@ -84,6 +85,7 @@ export function getProgram(userLog: LogFn): Command { .option('-p, --pxe [options]', 'Starts Aztec PXE with options (e.g. "option1=value1,option2=value2")') .option('-a, --archiver [options]', 'Starts Aztec Archiver attached to node') .option('-s, --sequencer [options]', 'Starts Aztec Sequencer attached to node') + .option('-p2p, --p2p-bootstrap', 'Starts a P2P bootstrap node') .action(async options => { // list of 'stop' functions to call when process ends const signalHandlers: Array<() => Promise> = []; @@ -182,6 +184,13 @@ export function getProgram(userLog: LogFn): Command { const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, true); startHttpRpcServer(archiver, createArchiverRpcServer, archiverCliOptions.port || ARCHIVER_PORT); + } else if (options.p2pBootstrap) { + // Start a P2P bootstrap node. + const envVars = getP2PConfigEnvVars(); + const cliOptions = parseModuleOptions(options.p2pBootstrap); + const bootstrapNode = new BootstrapNode(debugLogger); + await bootstrapNode.start({ ...envVars, ...cliOptions }); + signalHandlers.push(bootstrapNode.stop); } installSignalHandlers(debugLogger, signalHandlers); }); From 68daf3dfce2d6cd2ac4bb344fcd286d5d01e1e62 Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 10 Jan 2024 16:50:23 +0000 Subject: [PATCH 05/50] command descriptions --- aztec-up/bin/aztec | 2 +- yarn-project/aztec/src/bin/cli.ts | 201 +----------- yarn-project/aztec/src/cli/cli.ts | 291 ++++++++++++++++++ yarn-project/aztec/src/cli/index.ts | 1 + yarn-project/aztec/src/cli/texts.ts | 70 +++++ yarn-project/aztec/src/sandbox.ts | 8 +- .../end-to-end/src/e2e_p2p_network.test.ts | 2 +- yarn-project/p2p/src/client/p2p_client.ts | 4 +- yarn-project/p2p/src/config.ts | 4 +- yarn-project/pxe/src/config/index.ts | 7 +- 10 files changed, 381 insertions(+), 209 deletions(-) create mode 100644 yarn-project/aztec/src/cli/cli.ts create mode 100644 yarn-project/aztec/src/cli/index.ts create mode 100644 yarn-project/aztec/src/cli/texts.ts diff --git a/aztec-up/bin/aztec b/aztec-up/bin/aztec index 30ef8a66fab..9bec57051b9 100755 --- a/aztec-up/bin/aztec +++ b/aztec-up/bin/aztec @@ -1,4 +1,4 @@ #!/usr/bin/env bash set -euo pipefail -$(dirname $0)/.aztec-run aztecprotocol/aztec-sandbox $@ \ No newline at end of file +$(dirname $0)/.aztec-run aztecprotocol/aztec-sandbox $@ diff --git a/yarn-project/aztec/src/bin/cli.ts b/yarn-project/aztec/src/bin/cli.ts index b59a24ae0ec..9da1de41f17 100644 --- a/yarn-project/aztec/src/bin/cli.ts +++ b/yarn-project/aztec/src/bin/cli.ts @@ -1,206 +1,13 @@ -import { Archiver, LMDBArchiverStore, createArchiverRpcServer, getConfigEnvVars } from '@aztec/archiver'; -import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; -import { createAztecNodeClient, fileURLToPath } from '@aztec/aztec.js'; -import { NULL_KEY } from '@aztec/ethereum'; -import { LogFn, createConsoleLogger, createDebugLogger } from '@aztec/foundation/log'; -import { openDb } from '@aztec/kv-store'; -import { BootstrapNode, getP2PConfigEnvVars } from '@aztec/p2p'; -import { createPXERpcServer, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; +import { createConsoleLogger, createDebugLogger } from '@aztec/foundation/log'; -import { Command } from 'commander'; -import { readFileSync } from 'fs'; -import pick from 'lodash.pick'; -import { dirname, resolve } from 'path'; -import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; - -import { MNEMONIC, createAztecNode, createAztecPXE, createSandbox, deployContractsToL1 } from '../sandbox.js'; -import { startHttpRpcServer } from '../server.js'; - -const { AZTEC_NODE_PORT = '80', PXE_PORT = '79', ARCHIVER_PORT = '80' } = process.env; +import { getProgram } from '../cli/index.js'; +const userLog = createConsoleLogger(); const debugLogger = createDebugLogger('aztec:cli'); -const installSignalHandlers = (logFn: LogFn, cb?: Array<() => Promise>) => { - const shutdown = async () => { - logFn('Shutting down...'); - if (cb) { - await Promise.all(cb); - } - process.exit(0); - }; - process.removeAllListeners('SIGINT'); - process.removeAllListeners('SIGTERM'); - process.once('SIGINT', shutdown); - process.once('SIGTERM', shutdown); -}; - -/** - * Parses a string of options into a key-value map. - * @param options - String of options in the format "option1=value1,option2=value2". - * @returns Key-value map of options. - */ -const parseModuleOptions = (options: string): Record => { - const optionsArray = options.split(','); - return optionsArray.reduce((acc, option) => { - const [key, value] = option.split('='); - return { ...acc, [key]: value }; - }, {}); -}; - -/** - * Returns commander program that defines the 'aztec' command line interface. - * @param userLog - log function for logging user output. - * @param debugLogger - logger for logging debug messages. - */ -export function getProgram(userLog: LogFn): Command { - const program = new Command(); - - const packageJsonPath = resolve(dirname(fileURLToPath(import.meta.url)), '../../package.json'); - const cliVersion: string = JSON.parse(readFileSync(packageJsonPath).toString()).version; - - program.name('aztec').description('Aztec command line interface').version(cliVersion); - - // Start complete Sandbox. - program - .command('sandbox', 'Starts Aztec sandbox') - .option('-p, --port ', 'Port to run sandbox on', AZTEC_NODE_PORT) - .option('-pp, --pxe-port ', 'Port to run PXE on', PXE_PORT) - .action(async options => { - const { node, pxe, stop } = await createSandbox(); - installSignalHandlers(stop); - - // Start Node and PXE JSON-RPC servers - startHttpRpcServer(node, createAztecNodeRpcServer, options.port); - userLog(`Aztec Node JSON-RPC Server listening on port ${options.port}`); - if (options.pxePort) { - startHttpRpcServer(pxe, createPXERpcServer, PXE_PORT); - userLog(`PXE JSON-RPC Server listening on port ${PXE_PORT}`); - } - }); - - // Start Aztec modules with options - program - .command('start', 'Starts Aztec modules') - .option('-n, --node [options]', 'Starts Aztec Node with options (e.g. "option1=value1,option2=value2")') - .option('-p, --pxe [options]', 'Starts Aztec PXE with options (e.g. "option1=value1,option2=value2")') - .option('-a, --archiver [options]', 'Starts Aztec Archiver attached to node') - .option('-s, --sequencer [options]', 'Starts Aztec Sequencer attached to node') - .option('-p2p, --p2p-bootstrap', 'Starts a P2P bootstrap node') - .action(async options => { - // list of 'stop' functions to call when process ends - const signalHandlers: Array<() => Promise> = []; - - // Start Aztec Node - if (options.node) { - // get env vars first - const aztecNodeConfigEnvVars = getNodeConfigEnvVars(); - // get config from options - const nodeCliOptions = parseModuleOptions(options.node); - // merge env vars and cli options - const nodeConfig = pick({ ...aztecNodeConfigEnvVars, ...nodeCliOptions }) as AztecNodeConfig; - - // if no publisher private key, then use MNEMONIC - if (!options.archiver) { - // expect archiver url in node config - const archiverUrl = nodeCliOptions.archiverUrl; - if (!archiverUrl) { - throw new Error('Archiver Service URL is required to start Aztec Node without --archiver option'); - } - nodeConfig.archiverUrl = archiverUrl; - } - - // Deploy contracts if needed - if (nodeCliOptions.deployContracts) { - let account; - if (nodeConfig.publisherPrivateKey === NULL_KEY) { - account = mnemonicToAccount(MNEMONIC); - } else { - account = privateKeyToAccount(nodeConfig.publisherPrivateKey); - } - await deployContractsToL1(nodeConfig, account); - } - - if (!options.sequencer) { - nodeConfig.disableSequencer = true; - } else if (nodeConfig.publisherPrivateKey === NULL_KEY) { - // If we have a sequencer, ensure there's a publisher private key set. - const hdAccount = mnemonicToAccount(MNEMONIC); - const privKey = hdAccount.getHdKey().privateKey; - nodeConfig.publisherPrivateKey = `0x${Buffer.from(privKey!).toString('hex')}`; - } - - // P2P enabled by default - nodeConfig.p2pEnabled = nodeCliOptions.p2pDisabled !== 'true'; - - // Create and start Aztec Node. - const node = await createAztecNode(nodeConfig); - startHttpRpcServer(node, createAztecNodeRpcServer, nodeCliOptions.port || AZTEC_NODE_PORT); - userLog(`Aztec Node JSON-RPC Server listening on port ${nodeCliOptions.port || AZTEC_NODE_PORT}`); - - // Create a PXE client that connects to the node. - if (options.pxe) { - const pxeCliOptions = parseModuleOptions(options.pxe); - const pxe = await createAztecPXE(node); - signalHandlers.push(pxe.stop); - - // Start PXE JSON-RPC server. - startHttpRpcServer(pxe, createPXERpcServer, pxeCliOptions.port || PXE_PORT); - } - - signalHandlers.push(node.stop); - } else if (options.pxe) { - // Starting a PXE with a remote node. - // get env vars first - const pxeConfigEnvVars = getPXEServiceConfig(); - // get config from options - const pxeCliOptions = parseModuleOptions(options.pxe); - - // throw if no Aztec Node URL is provided - if (!pxeCliOptions.nodeUrl) { - throw new Error('Aztec Node URL (nodeUrl) option is required to start PXE without --node option'); - } - - // merge env vars and cli options - const pxeConfig = pick({ ...pxeConfigEnvVars, ...pxeCliOptions }); - - const node = createAztecNodeClient(pxeCliOptions.nodeUrl); - - const pxe = await createPXEService(node, pxeConfig); - signalHandlers.push(pxe.stop); - // Start PXE JSON-RPC server - startHttpRpcServer(pxe, createPXERpcServer, pxeCliOptions.port || PXE_PORT); - userLog(`PXE JSON-RPC Server listening on port ${pxeCliOptions.port || PXE_PORT}`); - } else if (options.archiver) { - // Start a standalone archiver. - // get env vars first - const archiverConfigEnvVars = getConfigEnvVars(); - // get config from options - const archiverCliOptions = parseModuleOptions(options.pxe); - // merge env vars and cli options - const archiverConfig = pick({ ...archiverConfigEnvVars, ...archiverCliOptions }); - - const [nodeDb] = await openDb(archiverConfig, debugLogger); - const archiverStore = new LMDBArchiverStore(nodeDb, archiverConfig.maxLogs); - - const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, true); - startHttpRpcServer(archiver, createArchiverRpcServer, archiverCliOptions.port || ARCHIVER_PORT); - } else if (options.p2pBootstrap) { - // Start a P2P bootstrap node. - const envVars = getP2PConfigEnvVars(); - const cliOptions = parseModuleOptions(options.p2pBootstrap); - const bootstrapNode = new BootstrapNode(debugLogger); - await bootstrapNode.start({ ...envVars, ...cliOptions }); - signalHandlers.push(bootstrapNode.stop); - } - installSignalHandlers(debugLogger, signalHandlers); - }); - return program; -} - /** CLI main entrypoint */ async function main() { - const userLog = createConsoleLogger(); - const program = getProgram(userLog); + const program = getProgram(userLog, debugLogger); await program.parseAsync(process.argv); } diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts new file mode 100644 index 00000000000..d6738662509 --- /dev/null +++ b/yarn-project/aztec/src/cli/cli.ts @@ -0,0 +1,291 @@ +import { Archiver, LMDBArchiverStore, createArchiverRpcServer, getConfigEnvVars } from '@aztec/archiver'; +import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; +import { + AccountManager, + GrumpkinScalar, + createAztecNodeClient, + deployInitialSandboxAccounts, + fileURLToPath, +} from '@aztec/aztec.js'; +import { NULL_KEY } from '@aztec/ethereum'; +import { DebugLogger, LogFn } from '@aztec/foundation/log'; +import { openDb } from '@aztec/kv-store'; +import { BootstrapNode, P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; +import { PXEService, PXEServiceConfig, createPXERpcServer, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; + +import { Command } from 'commander'; +import { readFileSync } from 'fs'; +import pick from 'lodash.pick'; +import { dirname, resolve } from 'path'; +import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; + +import { MNEMONIC, createAztecNode, createAztecPXE, createSandbox, deployContractsToL1 } from '../sandbox.js'; +import { startHttpRpcServer } from '../server.js'; +import { github, splash } from '../splash.js'; +import { cliTexts } from './texts.js'; + +const { + AZTEC_NODE_PORT = '80', + PXE_PORT = '79', + ARCHIVER_PORT = '80', + AZTEC_NODE_URL, + DEPLOY_AZTEC_CONTRACTS, +} = process.env; + +const installSignalHandlers = (logFn: LogFn, cb?: Array<() => Promise>) => { + const shutdown = async () => { + logFn('Shutting down...'); + if (cb) { + await Promise.all(cb); + } + process.exit(0); + }; + process.removeAllListeners('SIGINT'); + process.removeAllListeners('SIGTERM'); + process.once('SIGINT', shutdown); + process.once('SIGTERM', shutdown); +}; + +/** + * Parses a string of options into a key-value map. + * @param options - String of options in the format "option1=value1,option2=value2". + * @returns Key-value map of options. + */ +const parseModuleOptions = (options: string): Record => { + if (!options?.length) { + return {}; + } + const optionsArray = options.split(','); + return optionsArray.reduce((acc, option) => { + const [key, value] = option.split('='); + return { ...acc, [key]: value }; + }, {}); +}; + +const mergeEnvVarsAndCliOptions = (envVars: Record, cliOptions: Record) => { + const cliOptionsContracts = { + rollupAddress: cliOptions.rollupAddress, + registryAddress: cliOptions.registryAddress, + inboxAddress: cliOptions.inboxAddress, + outboxAddress: cliOptions.outboxAddress, + contractDeploymentEmitterAddress: cliOptions.contractDeploymentEmitterAddress, + }; + const merged = { + ...envVars, + ...cliOptions, + l1Contracts: { + ...envVars.l1Contracts, + ...cliOptionsContracts, + }, + } as T; + return merged; +}; + +/** + * Returns commander program that defines the 'aztec' command line interface. + * @param userLog - log function for logging user output. + * @param debugLogger - logger for logging debug messages. + */ +export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { + const program = new Command(); + + const packageJsonPath = resolve(dirname(fileURLToPath(import.meta.url)), '../../package.json'); + const cliVersion: string = JSON.parse(readFileSync(packageJsonPath).toString()).version; + + program.name('aztec').description('Aztec command line interface').version(cliVersion); + + // Start complete Sandbox. + program + .command('sandbox') + .description('Starts Aztec sandbox.') + .option('-p, --port ', 'Port to run sandbox on.', AZTEC_NODE_PORT) + .option('-pp, --pxe-port ', 'Port to run PXE on (optional).') + .option('-s, --skip-test-accounts', 'DO NOT deploy test accounts.', true) + // .option() + .action(async options => { + userLog(`${splash}\n${github}\n\n`); + userLog(`Setting up Aztec Sandbox v${cliVersion}, please stand by...`); + const { aztecNodeConfig, node, pxe, stop } = await createSandbox(); + installSignalHandlers(stop); + + // Deploy test accounts by default + if (!options.skipTestAccounts) { + if (aztecNodeConfig.p2pEnabled) { + userLog(`Not setting up test accounts as we are connecting to a network`); + } else { + userLog('Setting up test accounts...'); + const accounts = await deployInitialSandboxAccounts(pxe); + const accLogs = await createAccountLogs(accounts, pxe); + userLog(accLogs.join('')); + } + } + + // Start Node and PXE JSON-RPC servers + startHttpRpcServer(node, createAztecNodeRpcServer, options.port); + userLog(`Aztec Node JSON-RPC Server listening on port ${options.port}`); + if (options.pxePort) { + startHttpRpcServer(pxe, createPXERpcServer, PXE_PORT); + userLog(`PXE JSON-RPC Server listening on port ${PXE_PORT}`); + } + }); + + // Start Aztec modules with options + program + .command('start') + .description( + 'Starts Aztec modules. Options for each module can be set as key-value pairs (e.g. "option1=value1,option2=value2") or as environment variables.', + ) + .option('-n, --node [options]', cliTexts.node) + .option('-p, --pxe [options]', cliTexts.pxe) + .option('-a, --archiver [options]', cliTexts.archiver) + .option('-s, --sequencer [options]', cliTexts.sequencer) + .option('-p2p, --p2p-bootstrap', cliTexts.p2pBootstrap) + .action(async options => { + // list of 'stop' functions to call when process ends + const signalHandlers: Array<() => Promise> = []; + + // Start Aztec Node + if (options.node) { + // get env vars first + const aztecNodeConfigEnvVars = getNodeConfigEnvVars(); + // get config from options + const nodeCliOptions = parseModuleOptions(options.node); + // merge env vars and cli options + let nodeConfig = mergeEnvVarsAndCliOptions(aztecNodeConfigEnvVars, nodeCliOptions); + + // if no publisher private key, then use MNEMONIC + if (!options.archiver) { + // expect archiver url in node config + const archiverUrl = nodeCliOptions.archiverUrl; + if (!archiverUrl) { + throw new Error('Archiver Service URL is required to start Aztec Node without --archiver option'); + } + nodeConfig.archiverUrl = archiverUrl; + } else { + const archiverCliOptions = parseModuleOptions(options.archiver); + nodeConfig = mergeEnvVarsAndCliOptions(aztecNodeConfigEnvVars, archiverCliOptions); + } + + // Deploy contracts if needed + if (nodeCliOptions.deployAztecContracts || DEPLOY_AZTEC_CONTRACTS === 'true') { + let account; + if (nodeConfig.publisherPrivateKey === NULL_KEY) { + account = mnemonicToAccount(MNEMONIC); + } else { + account = privateKeyToAccount(nodeConfig.publisherPrivateKey); + } + await deployContractsToL1(nodeConfig, account); + } + + if (!options.sequencer) { + nodeConfig.disableSequencer = true; + } else if (nodeConfig.publisherPrivateKey === NULL_KEY) { + // If we have a sequencer, ensure there's a publisher private key set. + const hdAccount = mnemonicToAccount(MNEMONIC); + const privKey = hdAccount.getHdKey().privateKey; + nodeConfig.publisherPrivateKey = `0x${Buffer.from(privKey!).toString('hex')}`; + } + + // Create and start Aztec Node. + const node = await createAztecNode(nodeConfig); + startHttpRpcServer(node, createAztecNodeRpcServer, nodeCliOptions.port || AZTEC_NODE_PORT); + userLog(`Aztec Node JSON-RPC Server listening on port ${nodeCliOptions.port || AZTEC_NODE_PORT}`); + + // Create a PXE client that connects to the node. + if (options.pxe) { + const pxeCliOptions = parseModuleOptions(options.pxe); + const pxe = await createAztecPXE(node); + signalHandlers.push(pxe.stop); + + // Start PXE JSON-RPC server. + startHttpRpcServer(pxe, createPXERpcServer, pxeCliOptions.port || PXE_PORT); + } + + signalHandlers.push(node.stop); + } else if (options.pxe) { + // Starting a PXE with a remote node. + // get env vars first + const pxeConfigEnvVars = getPXEServiceConfig(); + // get config from options + const pxeCliOptions = parseModuleOptions(options.pxe); + + // Determine node url from options or env vars + const nodeUrl = pxeCliOptions.nodeUrl || AZTEC_NODE_URL; + // throw if no Aztec Node URL is provided + if (!nodeUrl) { + throw new Error( + 'Aztec Node URL (nodeUrl | AZTEC_NODE_URL) option is required to start PXE without --node option', + ); + } + + // merge env vars and cli options + const pxeConfig = mergeEnvVarsAndCliOptions(pxeConfigEnvVars, pxeCliOptions); + + // create a node client + const node = createAztecNodeClient(nodeUrl); + + const pxe = await createPXEService(node, pxeConfig); + signalHandlers.push(pxe.stop); + // Start PXE JSON-RPC server + startHttpRpcServer(pxe, createPXERpcServer, pxeCliOptions.port || PXE_PORT); + userLog(`PXE JSON-RPC Server listening on port ${pxeCliOptions.port || PXE_PORT}`); + } else if (options.archiver) { + // Start a standalone archiver. + // get env vars first + const archiverConfigEnvVars = getConfigEnvVars(); + // get config from options + const archiverCliOptions = parseModuleOptions(options.pxe); + // merge env vars and cli options + const archiverConfig = pick({ ...archiverConfigEnvVars, ...archiverCliOptions }); + + const [nodeDb] = await openDb(archiverConfig, debugLogger); + const archiverStore = new LMDBArchiverStore(nodeDb, archiverConfig.maxLogs); + + const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, true); + startHttpRpcServer(archiver, createArchiverRpcServer, archiverCliOptions.port || ARCHIVER_PORT); + } else if (options.p2pBootstrap) { + // Start a P2P bootstrap node. + const envVars = getP2PConfigEnvVars(); + const cliOptions = parseModuleOptions(options.p2pBootstrap); + const bootstrapNode = new BootstrapNode(debugLogger); + const config = mergeEnvVarsAndCliOptions(envVars, cliOptions); + await bootstrapNode.start(config); + signalHandlers.push(bootstrapNode.stop); + } + installSignalHandlers(debugLogger, signalHandlers); + }); + return program; +} + +/** + * Creates logs for the initial accounts + * @param accounts - The initial accounts + * @param pxe - A PXE instance to get the registered accounts + * @returns A string array containing the initial accounts details + */ +async function createAccountLogs( + accounts: { + /** + * The account object + */ + account: AccountManager; + /** + * The private key of the account + */ + privateKey: GrumpkinScalar; + }[], + pxe: PXEService, +) { + const registeredAccounts = await pxe.getRegisteredAccounts(); + const accountLogStrings = [`Initial Accounts:\n\n`]; + for (const account of accounts) { + const completeAddress = account.account.getCompleteAddress(); + if (registeredAccounts.find(a => a.equals(completeAddress))) { + accountLogStrings.push(` Address: ${completeAddress.address.toString()}\n`); + accountLogStrings.push(` Partial Address: ${completeAddress.partialAddress.toString()}\n`); + accountLogStrings.push(` Private Key: ${account.privateKey.toString()}\n`); + accountLogStrings.push(` Public Key: ${completeAddress.publicKey.toString()}\n\n`); + } + } + return accountLogStrings; +} diff --git a/yarn-project/aztec/src/cli/index.ts b/yarn-project/aztec/src/cli/index.ts new file mode 100644 index 00000000000..c01c2dcf0ba --- /dev/null +++ b/yarn-project/aztec/src/cli/index.ts @@ -0,0 +1 @@ +export * from './cli.js'; diff --git a/yarn-project/aztec/src/cli/texts.ts b/yarn-project/aztec/src/cli/texts.ts new file mode 100644 index 00000000000..38cdbb57f06 --- /dev/null +++ b/yarn-project/aztec/src/cli/texts.ts @@ -0,0 +1,70 @@ +const contractAddresses = + 'Aztec Contract Addresses:\n' + + 'rollupAddress:ROLLUP_CONTRACT_ADDRESS - string - The deployed L1 rollup contract address.\n' + + 'registryAddress:REGISTRY_CONTRACT_ADDRESS - string - The deployed L1 registry contract address.\n' + + 'inboxAddress:INBOX_CONTRACT_ADDRESS - string - The deployed L1 inbox contract address.\n' + + 'outboxAddress:OUTBOX_CONTRACT_ADDRESS - string - The deployed L1 outbox contract address.\n' + + 'contractDeploymentEmitterAddress:CONTRACT_DEPLOYMENT_EMITTER_ADDRESS - string - The deployed L1 contract deployment emitter contract address.\n'; +const p2pOptions = + 'p2pBlockCheckIntervalMS:P2P_BLOCK_CHECK_INTERVAL_MS - number - The frequency in which to check for blocks. Default: 100\n' + + 'p2pL2QueueSize:P2P_L2_QUEUE_SIZE - number - Size of queue of L2 blocks to store. Default: 1000\n' + + 'tcpListenPort:TCP_LISTEN_PORT - number - The tcp port on which the P2P service should listen for connections. Default: 40400\n' + + 'tcpListenIp:TCP_LISTEN_IP - string - The tcp IP on which the P2P service should listen for connections. Default: 0.0.0.0\n' + + 'peerIdPrivateKey:PEER_ID_PRIVATE_KEY - string - An optional peer id private key. If blank, will generate a random key.\n' + + 'bootstrapNodes:BOOTSTRAP_NODES - string - A list of bootstrap peers to connect to.\n' + + 'announceHostname:P2P_ANNOUNCE_HOSTNAME - string - P2P Hostname to announce.\n' + + 'announcePort:P2P_ANNOUNCE_PORT - number - P2P Port to announce.\n' + + 'clientKADRouting:P2P_KAD_CLIENT - boolean - Optional specification to run as a client in the Kademlia routing protocol. Default: false\n' + + 'enableNat:P2P_NAT_ENABLED - boolean - Whether to enable NAT from libp2p (ignored for bootstrap node). Default: false\n' + + 'minPeerCount:P2P_MIN_PEERS - number - The minimum number of peers to connect to. Default: 10\n' + + 'maxPeerCount:P2P_MAX_PEERS - number - The maximum number of peers to connect to. Default: 100\n'; + +export const cliTexts = { + node: + 'Starts Aztec Node with options.\n' + + 'Available options are listed below as cliProperty:ENV_VARIABLE_NAME.\n' + + 'rcpUrl:ETHEREUM_HOST - string - The host of the Ethereum node to connect to. Default: http://localhost:8545\n' + + 'archiverUrl:ARCHIVER_URL - string - A URL for an archiver service that the node will use.\n' + + 'p2pEnabled:P2P_ENABLED - boolean - A flag dictating whether the P2P subsystem should be enabled.\n\n' + + 'deployAztecContracts:DEPLOY_AZTEC_CONTRACTS - boolean - A flag dictating whether to deploy the Aztec contracts. Default: false\n' + + 'l2QueueSize:L2_QUEUE_SIZE - number - Size of queue of L2 blocks to store. Default: 1000\n' + + 'worldStateBlockCheckIntervalMS:WS_BLOCK_CHECK_INTERVAL_MS - number - The frequency in which to check for blocks in ms. Default: 100\n\n' + + // Contract Addresses + contractAddresses + + // P2P Options + 'When P2P is enabled, the following options are available:\n' + + p2pOptions, + pxe: + 'Starts a PXE with options. If started additionally to --node, the PXE will attach to that node.' + + 'Available options are listed below as cliProperty:ENV_VARIABLE_NAME.\n' + + 'nodeUrl:AZTEC_NODE_URL - string - The URL of the Aztec Node to connect to.\n' + + 'port:PXE_PORT - number - The port on which the PXE should listen for connections. Default: 79\n' + + 'l2BlockPollingIntervalMS:PXE_BLOCK_POLLING_INTERVAL_MS - number - The frequency in which to check for blocks in ms. Default: 1000\n' + + 'l2StartingBlock:PXE_L2_STARTING_BLOCK - number - The block number from which to start polling. Default: 1\n' + + 'dataDirectory:PXE_DATA_DIRECTORY - string - Where to store PXE data. If not set, will store in memory.\n', + archiver: + 'Starts an Archiver with options. If started additionally to --node, the Archiver will attach to that node.' + + 'Available options are listed below as cliProperty:ENV_VARIABLE_NAME.\n' + + 'rcpUrl:ETHEREUM_HOST - string - The host of the Ethereum node to connect to. Default: http://localhost:8545\n' + + 'apiKey:API_KEY - string - The key for the ethereum node if necessary.\n' + + 'archiverPollingIntervalMS:ARCHIVER_POLLING_INTERVAL_MS - number - The polling interval in ms for retrieving new L2 blocks and encrypted logs. Default: 1000\n' + + 'viemPollingIntervalMS:ARCHIVER_VIEM_POLLING_INTERVAL_MS - number - The polling interval viem uses in ms. Default: 1000\n' + + 'dataDirectory:DATA_DIRECTORY - string - Optional dir to store data. If omitted will store in memory.\n\n' + + contractAddresses, + sequencer: + 'Starts a Sequencer with options. If started additionally to --node, the Sequencer will attach to that node.\n' + + 'Available options are listed below as cliProperty:ENV_VARIABLE_NAME.\n' + + 'rcpUrl:ETHEREUM_HOST - string - The host of the Ethereum node to connect to. Default: http://localhost:8545\n' + + 'apiKey:API_KEY - string - The key for the ethereum node if necessary.\n' + + 'chainId:CHAIN_ID - number - The chain id of the ethereum host. Default: 31337\n' + + 'version:VERSION - number - The version of the Aztec rollup. Default: 1\n' + + 'publisherPrivateKey:SEQ_PUBLISHER_PRIVATE_KEY - string - The private key of the publisher. If not provided, will try to infer from default foundry test accounts.\n' + + 'requiredConfirmations:SEQ_REQUIRED_CONFIRMATIONS - number - The number of confirmations required before publishing a block. Default: 1\n' + + 'l1BlockPublishRetryIntervalMS:SEQ_PUBLISH_RETRY_INTERVAL_MS - number - The interval in ms to wait before retrying to publish a block. Default: 1000\n' + + 'transactionPollingIntervalMS:SEQ_TX_POLLING_INTERVAL_MS - number - The interval in ms to wait before polling for new transactions. Default: 1000\n\n' + + contractAddresses, + p2pBootstrap: + 'Starts a P2P bootstrap node with options.\n' + + 'Available options are listed below as cliProperty:ENV_VARIABLE_NAME.\n' + + p2pOptions, +}; diff --git a/yarn-project/aztec/src/sandbox.ts b/yarn-project/aztec/src/sandbox.ts index 13b04725068..ee12451345b 100644 --- a/yarn-project/aztec/src/sandbox.ts +++ b/yarn-project/aztec/src/sandbox.ts @@ -72,7 +72,11 @@ async function waitThenDeploy(config: AztecNodeConfig, deployFunction: () => Pro * @param aztecNodeConfig - The Aztec Node Config * @param hdAccount - Account for publishing L1 contracts */ -export async function deployContractsToL1(aztecNodeConfig: AztecNodeConfig, hdAccount: HDAccount | PrivateKeyAccount) { +export async function deployContractsToL1( + aztecNodeConfig: AztecNodeConfig, + hdAccount: HDAccount | PrivateKeyAccount, + contractDeployLogger = logger, +) { const l1Artifacts: L1ContractArtifactsForDeployment = { contractDeploymentEmitter: { contractAbi: ContractDeploymentEmitterAbi, @@ -98,7 +102,7 @@ export async function deployContractsToL1(aztecNodeConfig: AztecNodeConfig, hdAc aztecNodeConfig.l1Contracts = ( await waitThenDeploy(aztecNodeConfig, () => - deployL1Contracts(aztecNodeConfig.rpcUrl, hdAccount, localAnvil, logger, l1Artifacts), + deployL1Contracts(aztecNodeConfig.rpcUrl, hdAccount, localAnvil, contractDeployLogger, l1Artifacts), ) ).l1ContractAddresses; diff --git a/yarn-project/end-to-end/src/e2e_p2p_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p_network.test.ts index a5888f24dcb..50056776340 100644 --- a/yarn-project/end-to-end/src/e2e_p2p_network.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p_network.test.ts @@ -102,7 +102,7 @@ describe('e2e_p2p_network', () => { // TODO: the following config options are not applicable to bootstrap nodes p2pBlockCheckIntervalMS: 1000, - l2QueueSize: 1, + p2pL2QueueSize: 1, transactionProtocol: '', bootstrapNodes: [''], }; diff --git a/yarn-project/p2p/src/client/p2p_client.ts b/yarn-project/p2p/src/client/p2p_client.ts index 99798622e1d..8848e8781e1 100644 --- a/yarn-project/p2p/src/client/p2p_client.ts +++ b/yarn-project/p2p/src/client/p2p_client.ts @@ -132,8 +132,8 @@ export class P2PClient implements P2P { private p2pService: P2PService, private log = createDebugLogger('aztec:p2p'), ) { - const { p2pBlockCheckIntervalMS: checkInterval, l2QueueSize } = getP2PConfigEnvVars(); - this.blockDownloader = new L2BlockDownloader(l2BlockSource, l2QueueSize, checkInterval); + const { p2pBlockCheckIntervalMS: checkInterval, p2pL2QueueSize } = getP2PConfigEnvVars(); + this.blockDownloader = new L2BlockDownloader(l2BlockSource, p2pL2QueueSize, checkInterval); this.synchedBlockNumber = store.createSingleton('p2p_pool_last_l2_block'); } diff --git a/yarn-project/p2p/src/config.ts b/yarn-project/p2p/src/config.ts index 8e4d4d1af94..a995bcb8195 100644 --- a/yarn-project/p2p/src/config.ts +++ b/yarn-project/p2p/src/config.ts @@ -15,7 +15,7 @@ export interface P2PConfig { /** * Size of queue of L2 blocks to store. */ - l2QueueSize: number; + p2pL2QueueSize: number; /** * The tcp port on which the P2P service should listen for connections. @@ -96,7 +96,7 @@ export function getP2PConfigEnvVars(): P2PConfig { const envVars: P2PConfig = { p2pEnabled: P2P_ENABLED === 'true', p2pBlockCheckIntervalMS: P2P_BLOCK_CHECK_INTERVAL_MS ? +P2P_BLOCK_CHECK_INTERVAL_MS : 100, - l2QueueSize: P2P_L2_BLOCK_QUEUE_SIZE ? +P2P_L2_BLOCK_QUEUE_SIZE : 1000, + p2pL2QueueSize: P2P_L2_BLOCK_QUEUE_SIZE ? +P2P_L2_BLOCK_QUEUE_SIZE : 1000, tcpListenPort: P2P_TCP_LISTEN_PORT ? +P2P_TCP_LISTEN_PORT : 40400, tcpListenIp: P2P_TCP_LISTEN_IP ? P2P_TCP_LISTEN_IP : '0.0.0.0', peerIdPrivateKey: PEER_ID_PRIVATE_KEY, diff --git a/yarn-project/pxe/src/config/index.ts b/yarn-project/pxe/src/config/index.ts index 7ad14c519c6..a72898a53db 100644 --- a/yarn-project/pxe/src/config/index.ts +++ b/yarn-project/pxe/src/config/index.ts @@ -12,8 +12,7 @@ export interface PXEServiceConfig { l2BlockPollingIntervalMS: number; /** L2 block to start scanning from for new accounts */ l2StartingBlock: number; - - /** Where to store PXE data. If not set will store in memory */ + /** Where to store PXE data. If not set, will store in memory */ dataDirectory?: string; } @@ -21,12 +20,12 @@ export interface PXEServiceConfig { * Creates an instance of PXEServiceConfig out of environment variables using sensible defaults for integration testing if not set. */ export function getPXEServiceConfig(): PXEServiceConfig { - const { PXE_BLOCK_POLLING_INTERVAL_MS, PXE_L2_STARTING_BLOCK, DATA_DIRECTORY } = process.env; + const { PXE_BLOCK_POLLING_INTERVAL_MS, PXE_L2_STARTING_BLOCK, PXE_DATA_DIRECTORY } = process.env; return { l2BlockPollingIntervalMS: PXE_BLOCK_POLLING_INTERVAL_MS ? +PXE_BLOCK_POLLING_INTERVAL_MS : 1000, l2StartingBlock: PXE_L2_STARTING_BLOCK ? +PXE_L2_STARTING_BLOCK : INITIAL_L2_BLOCK_NUM, - dataDirectory: DATA_DIRECTORY, + dataDirectory: PXE_DATA_DIRECTORY, }; } From 5dc0208b041c78cbef0e2b686d8e6ce8553bef91 Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 11 Jan 2024 09:43:39 +0000 Subject: [PATCH 06/50] typing fixes --- aztec-up/bin/aztec | 2 +- aztec-up/bin/aztec-install | 6 +- yarn-project/aztec/src/bin/_index.ts | 245 ++++++++++++++++++ yarn-project/aztec/src/bin/cli.ts | 18 -- yarn-project/aztec/src/bin/index.ts | 245 +----------------- yarn-project/aztec/src/cli/cli.ts | 13 +- .../end-to-end/src/fixtures/fixtures.ts | 3 - yarn-project/end-to-end/src/fixtures/utils.ts | 9 +- .../p2p/src/service/libp2p_service.ts | 2 +- .../pxe/src/pxe_http/pxe_http_server.ts | 3 - yarn-project/yarn.lock | 38 +-- 11 files changed, 287 insertions(+), 297 deletions(-) create mode 100644 yarn-project/aztec/src/bin/_index.ts delete mode 100644 yarn-project/aztec/src/bin/cli.ts diff --git a/aztec-up/bin/aztec b/aztec-up/bin/aztec index 9bec57051b9..31e43f651be 100755 --- a/aztec-up/bin/aztec +++ b/aztec-up/bin/aztec @@ -1,4 +1,4 @@ #!/usr/bin/env bash set -euo pipefail -$(dirname $0)/.aztec-run aztecprotocol/aztec-sandbox $@ +$(dirname $0)/.aztec-run aztecprotocol/aztec $@ diff --git a/aztec-up/bin/aztec-install b/aztec-up/bin/aztec-install index a49c37fefff..3c7dec94844 100755 --- a/aztec-up/bin/aztec-install +++ b/aztec-up/bin/aztec-install @@ -40,7 +40,7 @@ function title() { echo -e "${r}" fi echo -e "This will install the following scripts and update your PATH if necessary:" - echo -e " ${bold}${g}aztec${r} - launches various infrastructure subsystems (sequencer, prover, pxe, etc)." + echo -e " ${bold}${g}aztec${r} - launches various infrastructure subsystems (node, sequencer, prover, pxe, etc)." echo -e " ${bold}${g}aztec-cli${r} - a command line tool for interfacing and experimenting with infrastructure." echo -e " ${bold}${g}aztec-nargo${r} - aztec's build of nargo, the noir compiler toolchain." echo -e " ${bold}${g}aztec-sandbox${r} - a wrapper around docker-compose that launches services needed for sandbox testing." @@ -106,12 +106,12 @@ export DOCKER_CLI_HINTS=false if [ -z "${SKIP_PULL:-}" ]; then info "Pulling aztec version $VERSION..." - pull_container aztec-sandbox + pull_container aztec pull_container cli pull_container noir fi -# Download the Docker Compose file. Used by aztec-sandbox. +# Download the Docker Compose file. Used by aztec. curl -fsSL http://$INSTALL_HOST/docker-compose.yml -o $AZTEC_PATH/docker-compose.yml function install_bin { diff --git a/yarn-project/aztec/src/bin/_index.ts b/yarn-project/aztec/src/bin/_index.ts new file mode 100644 index 00000000000..75b8d311347 --- /dev/null +++ b/yarn-project/aztec/src/bin/_index.ts @@ -0,0 +1,245 @@ +#!/usr/bin/env -S node --no-warnings +import { deployInitialTestAccounts } from '@aztec/accounts/testing'; +import { createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; +import { AccountManager, createAztecNodeClient } from '@aztec/aztec.js'; +import { NULL_KEY } from '@aztec/ethereum'; +import { init } from '@aztec/foundation/crypto'; +import { createStatusRouter, startHttpRpcServer } from '@aztec/foundation/json-rpc/server'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { fileURLToPath } from '@aztec/foundation/url'; +import { NoirCommit, NoirTag } from '@aztec/noir-compiler/versions'; +import { BootstrapNode, getP2PConfigEnvVars } from '@aztec/p2p'; +import { GrumpkinScalar, PXEService, createPXERpcServer } from '@aztec/pxe'; + +import { lookup } from 'dns/promises'; +import { readFileSync } from 'fs'; +import http from 'http'; +import { dirname, resolve } from 'path'; +import { mnemonicToAccount } from 'viem/accounts'; + +import { setupFileDebugLog } from '../logging.js'; +import { MNEMONIC, createAztecNode, createAztecPXE, createSandbox, deployContractsToL1 } from '../sandbox.js'; +import { github, splash } from '../splash.js'; + +/** + * The mode in which the sandbox should be run. + */ +enum SandboxMode { + Sandbox = 'sandbox', + Node = 'node', + PXE = 'pxe', + P2PBootstrap = 'p2p-bootstrap', +} + +/** + * If we can successfully resolve 'host.docker.internal', then we are running in a container, and we should treat + * localhost as being host.docker.internal. + */ +const getLocalhost = () => + lookup('host.docker.internal') + .then(() => 'host.docker.internal') + .catch(() => 'localhost'); + +const LOCALHOST = await getLocalhost(); +const { + AZTEC_NODE_URL = `http://${LOCALHOST}:8079`, + AZTEC_NODE_PORT = 8079, + PXE_PORT = 8080, + MODE = 'sandbox', + TEST_ACCOUNTS = 'true', + DEPLOY_AZTEC_CONTRACTS = 'true', + API_PREFIX = '', +} = process.env; + +const logger = createDebugLogger(`aztec:${MODE}`); + +/** + * Creates the sandbox from provided config and deploys any initial L1 and L2 contracts + */ +async function createAndInitialiseSandbox(deployTestAccounts: boolean) { + const { aztecNodeConfig, node, pxe, stop } = await createSandbox(); + if (aztecNodeConfig.p2pEnabled) { + logger.info(`Not setting up test accounts as we are connecting to a network`); + return { + aztecNodeConfig, + pxe, + node, + stop, + accounts: [], + }; + } + let accounts; + if (deployTestAccounts) { + logger.info('Setting up test accounts...'); + accounts = await deployInitialTestAccounts(pxe); + } + return { + aztecNodeConfig, + pxe, + node, + stop, + accounts, + }; +} + +/** + * Create and start a new Aztec RPC HTTP Server + */ +async function main() { + const deployTestAccounts = TEST_ACCOUNTS === 'true'; + const deployAztecContracts = DEPLOY_AZTEC_CONTRACTS === 'true'; + + const mode = MODE as SandboxMode; + + const installSignalHandlers = (cb?: () => Promise) => { + const shutdown = async () => { + logger.info('Shutting down...'); + if (cb) { + await cb(); + } + process.exit(0); + }; + process.removeAllListeners('SIGINT'); + process.removeAllListeners('SIGTERM'); + process.once('SIGINT', shutdown); + process.once('SIGTERM', shutdown); + }; + + installSignalHandlers(); + + // Init crypto (bb.js). + await init(); + + const logStrings = []; + + const logPath = setupFileDebugLog(); + logger.info(`Debug logs will be written to ${logPath}`); + + // Get Sandbox version + const packageJsonPath = resolve(dirname(fileURLToPath(import.meta.url)), '../../package.json'); + const version = JSON.parse(readFileSync(packageJsonPath).toString()).version; + + // Code path for starting Sandbox + if (mode === SandboxMode.Sandbox) { + logger.info(`Setting up Aztec Sandbox v${version} (noir ${NoirCommit} ${NoirTag}), please stand by...`); + + const { pxe, node, stop, accounts } = await createAndInitialiseSandbox(deployTestAccounts); + + // Create shutdown cleanup function + installSignalHandlers(stop); + + // Start Node and PXE JSON-RPC servers + startHttpRpcServer(node, createAztecNodeRpcServer, AZTEC_NODE_PORT); + logger.info(`Aztec Node JSON-RPC Server listening on port ${AZTEC_NODE_PORT}`); + startHttpRpcServer(pxe, createPXERpcServer, PXE_PORT); + logger.info(`PXE JSON-RPC Server listening on port ${PXE_PORT}`); + + // Log initial accounts details + if (accounts?.length) { + const accountLogStrings = await createAccountLogs(accounts, pxe); + logStrings.push(...accountLogStrings); + } + logStrings.push(`Aztec Sandbox v${version} (noir ${NoirCommit}) is now ready for use!`); + } else if (mode === SandboxMode.Node) { + // Code path for starting Node only + const nodeConfig = getNodeConfigEnvVars(); + const hdAccount = mnemonicToAccount(MNEMONIC); + + // Deploy L1 Aztec Contracts if needed + if (deployAztecContracts) { + await deployContractsToL1(nodeConfig, hdAccount); + if (nodeConfig.publisherPrivateKey === NULL_KEY) { + const privKey = hdAccount.getHdKey().privateKey; + nodeConfig.publisherPrivateKey = `0x${Buffer.from(privKey!).toString('hex')}`; + } + } + + const node = await createAztecNode(nodeConfig); + installSignalHandlers(node.stop); + + const port = process.env.AZTEC_NODE_PORT || 8080; // Use standard 8080 when no PXE is running + const nodeRpcServer = createAztecNodeRpcServer(node); + const app = nodeRpcServer.getApp(API_PREFIX); + + // Add a /status endpoint + const statusRouter = createStatusRouter(API_PREFIX); + app.use(statusRouter.routes()); + app.use(statusRouter.allowedMethods()); + + // Start Node JSON-RPC server + const httpServer = http.createServer(app.callback()); + httpServer.listen(port); + + logStrings.push(`Aztec Node v${version} (noir ${NoirCommit}) is now ready for use in port ${port}!`); + } else if (mode === SandboxMode.PXE) { + // Code path for starting PXE only + + // Create a Node client to connect to the PXE + const node = createAztecNodeClient(AZTEC_NODE_URL); + + const pxe = await createAztecPXE(node); + installSignalHandlers(pxe.stop); + + // Start PXE JSON-RPC server + startHttpRpcServer(pxe, createPXERpcServer, PXE_PORT); + + if (deployTestAccounts) { + logger.info('Setting up test accounts...'); + const accounts = await deployInitialTestAccounts(pxe); + const accountLogStrings = await createAccountLogs(accounts, pxe); + logStrings.push(...accountLogStrings); + } + + logStrings.push(`PXE v${version} (noir ${NoirCommit}) is now ready for use in port ${PXE_PORT}!`); + } else if (mode === SandboxMode.P2PBootstrap) { + // Code path for starting a P2P bootstrap node + const config = getP2PConfigEnvVars(); + const bootstrapNode = new BootstrapNode(logger); + await bootstrapNode.start(config); + installSignalHandlers(bootstrapNode.stop); + logStrings.push( + `Bootstrap P2P node is now ready for use. Listening on: ${config.tcpListenIp}:${config.tcpListenPort}.`, + ); + } + + // Log startup details + logger.info(`${splash}\n${github}\n\n`.concat(...logStrings)); +} + +/** + * Creates logs for the initial accounts + * @param accounts - The initial accounts + * @param pxe - A PXE instance to get the registered accounts + * @returns A string array containing the initial accounts details + */ +async function createAccountLogs( + accounts: { + /** + * The account object + */ + account: AccountManager; + /** + * The private key of the account + */ + privateKey: GrumpkinScalar; + }[], + pxe: PXEService, +) { + const registeredAccounts = await pxe.getRegisteredAccounts(); + const accountLogStrings = [`Initial Accounts:\n\n`]; + for (const account of accounts) { + const completeAddress = account.account.getCompleteAddress(); + if (registeredAccounts.find(a => a.equals(completeAddress))) { + accountLogStrings.push(` Address: ${completeAddress.address.toString()}\n`); + accountLogStrings.push(` Partial Address: ${completeAddress.partialAddress.toString()}\n`); + accountLogStrings.push(` Private Key: ${account.privateKey.toString()}\n`); + accountLogStrings.push(` Public Key: ${completeAddress.publicKey.toString()}\n\n`); + } + } + return accountLogStrings; +} + +main().catch(err => { + logger.error(err); + process.exit(1); +}); diff --git a/yarn-project/aztec/src/bin/cli.ts b/yarn-project/aztec/src/bin/cli.ts deleted file mode 100644 index 9da1de41f17..00000000000 --- a/yarn-project/aztec/src/bin/cli.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { createConsoleLogger, createDebugLogger } from '@aztec/foundation/log'; - -import { getProgram } from '../cli/index.js'; - -const userLog = createConsoleLogger(); -const debugLogger = createDebugLogger('aztec:cli'); - -/** CLI main entrypoint */ -async function main() { - const program = getProgram(userLog, debugLogger); - await program.parseAsync(process.argv); -} - -main().catch(err => { - debugLogger(`Error in command execution`); - debugLogger(err); - process.exit(1); -}); diff --git a/yarn-project/aztec/src/bin/index.ts b/yarn-project/aztec/src/bin/index.ts index 75b8d311347..9da1de41f17 100644 --- a/yarn-project/aztec/src/bin/index.ts +++ b/yarn-project/aztec/src/bin/index.ts @@ -1,245 +1,18 @@ -#!/usr/bin/env -S node --no-warnings -import { deployInitialTestAccounts } from '@aztec/accounts/testing'; -import { createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; -import { AccountManager, createAztecNodeClient } from '@aztec/aztec.js'; -import { NULL_KEY } from '@aztec/ethereum'; -import { init } from '@aztec/foundation/crypto'; -import { createStatusRouter, startHttpRpcServer } from '@aztec/foundation/json-rpc/server'; -import { createDebugLogger } from '@aztec/foundation/log'; -import { fileURLToPath } from '@aztec/foundation/url'; -import { NoirCommit, NoirTag } from '@aztec/noir-compiler/versions'; -import { BootstrapNode, getP2PConfigEnvVars } from '@aztec/p2p'; -import { GrumpkinScalar, PXEService, createPXERpcServer } from '@aztec/pxe'; +import { createConsoleLogger, createDebugLogger } from '@aztec/foundation/log'; -import { lookup } from 'dns/promises'; -import { readFileSync } from 'fs'; -import http from 'http'; -import { dirname, resolve } from 'path'; -import { mnemonicToAccount } from 'viem/accounts'; +import { getProgram } from '../cli/index.js'; -import { setupFileDebugLog } from '../logging.js'; -import { MNEMONIC, createAztecNode, createAztecPXE, createSandbox, deployContractsToL1 } from '../sandbox.js'; -import { github, splash } from '../splash.js'; +const userLog = createConsoleLogger(); +const debugLogger = createDebugLogger('aztec:cli'); -/** - * The mode in which the sandbox should be run. - */ -enum SandboxMode { - Sandbox = 'sandbox', - Node = 'node', - PXE = 'pxe', - P2PBootstrap = 'p2p-bootstrap', -} - -/** - * If we can successfully resolve 'host.docker.internal', then we are running in a container, and we should treat - * localhost as being host.docker.internal. - */ -const getLocalhost = () => - lookup('host.docker.internal') - .then(() => 'host.docker.internal') - .catch(() => 'localhost'); - -const LOCALHOST = await getLocalhost(); -const { - AZTEC_NODE_URL = `http://${LOCALHOST}:8079`, - AZTEC_NODE_PORT = 8079, - PXE_PORT = 8080, - MODE = 'sandbox', - TEST_ACCOUNTS = 'true', - DEPLOY_AZTEC_CONTRACTS = 'true', - API_PREFIX = '', -} = process.env; - -const logger = createDebugLogger(`aztec:${MODE}`); - -/** - * Creates the sandbox from provided config and deploys any initial L1 and L2 contracts - */ -async function createAndInitialiseSandbox(deployTestAccounts: boolean) { - const { aztecNodeConfig, node, pxe, stop } = await createSandbox(); - if (aztecNodeConfig.p2pEnabled) { - logger.info(`Not setting up test accounts as we are connecting to a network`); - return { - aztecNodeConfig, - pxe, - node, - stop, - accounts: [], - }; - } - let accounts; - if (deployTestAccounts) { - logger.info('Setting up test accounts...'); - accounts = await deployInitialTestAccounts(pxe); - } - return { - aztecNodeConfig, - pxe, - node, - stop, - accounts, - }; -} - -/** - * Create and start a new Aztec RPC HTTP Server - */ +/** CLI main entrypoint */ async function main() { - const deployTestAccounts = TEST_ACCOUNTS === 'true'; - const deployAztecContracts = DEPLOY_AZTEC_CONTRACTS === 'true'; - - const mode = MODE as SandboxMode; - - const installSignalHandlers = (cb?: () => Promise) => { - const shutdown = async () => { - logger.info('Shutting down...'); - if (cb) { - await cb(); - } - process.exit(0); - }; - process.removeAllListeners('SIGINT'); - process.removeAllListeners('SIGTERM'); - process.once('SIGINT', shutdown); - process.once('SIGTERM', shutdown); - }; - - installSignalHandlers(); - - // Init crypto (bb.js). - await init(); - - const logStrings = []; - - const logPath = setupFileDebugLog(); - logger.info(`Debug logs will be written to ${logPath}`); - - // Get Sandbox version - const packageJsonPath = resolve(dirname(fileURLToPath(import.meta.url)), '../../package.json'); - const version = JSON.parse(readFileSync(packageJsonPath).toString()).version; - - // Code path for starting Sandbox - if (mode === SandboxMode.Sandbox) { - logger.info(`Setting up Aztec Sandbox v${version} (noir ${NoirCommit} ${NoirTag}), please stand by...`); - - const { pxe, node, stop, accounts } = await createAndInitialiseSandbox(deployTestAccounts); - - // Create shutdown cleanup function - installSignalHandlers(stop); - - // Start Node and PXE JSON-RPC servers - startHttpRpcServer(node, createAztecNodeRpcServer, AZTEC_NODE_PORT); - logger.info(`Aztec Node JSON-RPC Server listening on port ${AZTEC_NODE_PORT}`); - startHttpRpcServer(pxe, createPXERpcServer, PXE_PORT); - logger.info(`PXE JSON-RPC Server listening on port ${PXE_PORT}`); - - // Log initial accounts details - if (accounts?.length) { - const accountLogStrings = await createAccountLogs(accounts, pxe); - logStrings.push(...accountLogStrings); - } - logStrings.push(`Aztec Sandbox v${version} (noir ${NoirCommit}) is now ready for use!`); - } else if (mode === SandboxMode.Node) { - // Code path for starting Node only - const nodeConfig = getNodeConfigEnvVars(); - const hdAccount = mnemonicToAccount(MNEMONIC); - - // Deploy L1 Aztec Contracts if needed - if (deployAztecContracts) { - await deployContractsToL1(nodeConfig, hdAccount); - if (nodeConfig.publisherPrivateKey === NULL_KEY) { - const privKey = hdAccount.getHdKey().privateKey; - nodeConfig.publisherPrivateKey = `0x${Buffer.from(privKey!).toString('hex')}`; - } - } - - const node = await createAztecNode(nodeConfig); - installSignalHandlers(node.stop); - - const port = process.env.AZTEC_NODE_PORT || 8080; // Use standard 8080 when no PXE is running - const nodeRpcServer = createAztecNodeRpcServer(node); - const app = nodeRpcServer.getApp(API_PREFIX); - - // Add a /status endpoint - const statusRouter = createStatusRouter(API_PREFIX); - app.use(statusRouter.routes()); - app.use(statusRouter.allowedMethods()); - - // Start Node JSON-RPC server - const httpServer = http.createServer(app.callback()); - httpServer.listen(port); - - logStrings.push(`Aztec Node v${version} (noir ${NoirCommit}) is now ready for use in port ${port}!`); - } else if (mode === SandboxMode.PXE) { - // Code path for starting PXE only - - // Create a Node client to connect to the PXE - const node = createAztecNodeClient(AZTEC_NODE_URL); - - const pxe = await createAztecPXE(node); - installSignalHandlers(pxe.stop); - - // Start PXE JSON-RPC server - startHttpRpcServer(pxe, createPXERpcServer, PXE_PORT); - - if (deployTestAccounts) { - logger.info('Setting up test accounts...'); - const accounts = await deployInitialTestAccounts(pxe); - const accountLogStrings = await createAccountLogs(accounts, pxe); - logStrings.push(...accountLogStrings); - } - - logStrings.push(`PXE v${version} (noir ${NoirCommit}) is now ready for use in port ${PXE_PORT}!`); - } else if (mode === SandboxMode.P2PBootstrap) { - // Code path for starting a P2P bootstrap node - const config = getP2PConfigEnvVars(); - const bootstrapNode = new BootstrapNode(logger); - await bootstrapNode.start(config); - installSignalHandlers(bootstrapNode.stop); - logStrings.push( - `Bootstrap P2P node is now ready for use. Listening on: ${config.tcpListenIp}:${config.tcpListenPort}.`, - ); - } - - // Log startup details - logger.info(`${splash}\n${github}\n\n`.concat(...logStrings)); -} - -/** - * Creates logs for the initial accounts - * @param accounts - The initial accounts - * @param pxe - A PXE instance to get the registered accounts - * @returns A string array containing the initial accounts details - */ -async function createAccountLogs( - accounts: { - /** - * The account object - */ - account: AccountManager; - /** - * The private key of the account - */ - privateKey: GrumpkinScalar; - }[], - pxe: PXEService, -) { - const registeredAccounts = await pxe.getRegisteredAccounts(); - const accountLogStrings = [`Initial Accounts:\n\n`]; - for (const account of accounts) { - const completeAddress = account.account.getCompleteAddress(); - if (registeredAccounts.find(a => a.equals(completeAddress))) { - accountLogStrings.push(` Address: ${completeAddress.address.toString()}\n`); - accountLogStrings.push(` Partial Address: ${completeAddress.partialAddress.toString()}\n`); - accountLogStrings.push(` Private Key: ${account.privateKey.toString()}\n`); - accountLogStrings.push(` Public Key: ${completeAddress.publicKey.toString()}\n\n`); - } - } - return accountLogStrings; + const program = getProgram(userLog, debugLogger); + await program.parseAsync(process.argv); } main().catch(err => { - logger.error(err); + debugLogger(`Error in command execution`); + debugLogger(err); process.exit(1); }); diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index d6738662509..44dfd1d1dfe 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -1,13 +1,9 @@ +import { deployInitialTestAccounts } from '@aztec/accounts/testing'; import { Archiver, LMDBArchiverStore, createArchiverRpcServer, getConfigEnvVars } from '@aztec/archiver'; import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; -import { - AccountManager, - GrumpkinScalar, - createAztecNodeClient, - deployInitialSandboxAccounts, - fileURLToPath, -} from '@aztec/aztec.js'; +import { AccountManager, GrumpkinScalar, createAztecNodeClient, fileURLToPath } from '@aztec/aztec.js'; import { NULL_KEY } from '@aztec/ethereum'; +import { startHttpRpcServer } from '@aztec/foundation/json-rpc/server'; import { DebugLogger, LogFn } from '@aztec/foundation/log'; import { openDb } from '@aztec/kv-store'; import { BootstrapNode, P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; @@ -20,7 +16,6 @@ import { dirname, resolve } from 'path'; import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; import { MNEMONIC, createAztecNode, createAztecPXE, createSandbox, deployContractsToL1 } from '../sandbox.js'; -import { startHttpRpcServer } from '../server.js'; import { github, splash } from '../splash.js'; import { cliTexts } from './texts.js'; @@ -114,7 +109,7 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { userLog(`Not setting up test accounts as we are connecting to a network`); } else { userLog('Setting up test accounts...'); - const accounts = await deployInitialSandboxAccounts(pxe); + const accounts = await deployInitialTestAccounts(pxe); const accLogs = await createAccountLogs(accounts, pxe); userLog(accLogs.join('')); } diff --git a/yarn-project/end-to-end/src/fixtures/fixtures.ts b/yarn-project/end-to-end/src/fixtures/fixtures.ts index f1bb81bea1e..3e4ffc7c8ae 100644 --- a/yarn-project/end-to-end/src/fixtures/fixtures.ts +++ b/yarn-project/end-to-end/src/fixtures/fixtures.ts @@ -1,6 +1,3 @@ -import { foundry } from 'viem/chains'; - export const MNEMONIC = 'test test test test test test test test test test test junk'; -export const localAnvil = foundry; export const privateKey = Buffer.from('ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80', 'hex'); export const privateKey2 = Buffer.from('59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d', 'hex'); diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index 9121ca0e114..64ebe0fdf0a 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -48,8 +48,9 @@ import { http, } from 'viem'; import { mnemonicToAccount } from 'viem/accounts'; +import { foundry } from 'viem/chains'; -import { MNEMONIC, localAnvil } from './fixtures.js'; +import { MNEMONIC } from './fixtures.js'; import { isMetricsLoggingRequested, setupMetricsLogger } from './logging.js'; export { deployAndInitializeTokenAndBridgeContracts } from '../shared/cross_chain_test_harness.js'; @@ -101,7 +102,7 @@ export const setupL1Contracts = async ( contractBytecode: DecoderHelperBytecode, }; } - return await deployL1Contracts(l1RpcUrl, account, localAnvil, logger, l1Artifacts); + return await deployL1Contracts(l1RpcUrl, account, foundry, logger, l1Artifacts); }; /** @@ -187,11 +188,11 @@ async function setupWithRemoteEnvironment( const walletClient = createWalletClient({ account, - chain: localAnvil, + chain: foundry, transport: http(config.rpcUrl), }); const publicClient = createPublicClient({ - chain: localAnvil, + chain: foundry, transport: http(config.rpcUrl), }); const deployL1ContractsValues: DeployL1Contracts = { diff --git a/yarn-project/p2p/src/service/libp2p_service.ts b/yarn-project/p2p/src/service/libp2p_service.ts index e13c7e18eda..eabb8aed14b 100644 --- a/yarn-project/p2p/src/service/libp2p_service.ts +++ b/yarn-project/p2p/src/service/libp2p_service.ts @@ -130,7 +130,7 @@ export class LibP2PService implements P2PService { */ public async stop() { if (this.timeout) { - clearTimeout(this.timeout); + clearTimeout(this.timeout as NodeJS.Timeout); } await this.jobQueue.end(); await this.node.stop(); diff --git a/yarn-project/pxe/src/pxe_http/pxe_http_server.ts b/yarn-project/pxe/src/pxe_http/pxe_http_server.ts index 0cde8a8b396..ce50ecee41d 100644 --- a/yarn-project/pxe/src/pxe_http/pxe_http_server.ts +++ b/yarn-project/pxe/src/pxe_http/pxe_http_server.ts @@ -23,9 +23,6 @@ import { } from '@aztec/types'; import http from 'http'; -import { foundry } from 'viem/chains'; - -export const localAnvil = foundry; /** * Wraps an instance of Private eXecution Environment (PXE) implementation to a JSON RPC HTTP interface. diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 76334b9ee97..0ac7f53bed8 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -1656,13 +1656,13 @@ __metadata: linkType: hard "@humanwhocodes/config-array@npm:^0.11.13": - version: 0.11.13 - resolution: "@humanwhocodes/config-array@npm:0.11.13" + version: 0.11.14 + resolution: "@humanwhocodes/config-array@npm:0.11.14" dependencies: - "@humanwhocodes/object-schema": ^2.0.1 - debug: ^4.1.1 + "@humanwhocodes/object-schema": ^2.0.2 + debug: ^4.3.1 minimatch: ^3.0.5 - checksum: f8ea57b0d7ed7f2d64cd3944654976829d9da91c04d9c860e18804729a33f7681f78166ef4c761850b8c324d362f7d53f14c5c44907a6b38b32c703ff85e4805 + checksum: 861ccce9eaea5de19546653bccf75bf09fe878bc39c3aab00aeee2d2a0e654516adad38dd1098aab5e3af0145bbcbf3f309bdf4d964f8dab9dcd5834ae4c02f2 languageName: node linkType: hard @@ -1673,10 +1673,10 @@ __metadata: languageName: node linkType: hard -"@humanwhocodes/object-schema@npm:^2.0.1": - version: 2.0.1 - resolution: "@humanwhocodes/object-schema@npm:2.0.1" - checksum: 24929487b1ed48795d2f08346a0116cc5ee4634848bce64161fb947109352c562310fd159fc64dda0e8b853307f5794605191a9547f7341158559ca3c8262a45 +"@humanwhocodes/object-schema@npm:^2.0.2": + version: 2.0.2 + resolution: "@humanwhocodes/object-schema@npm:2.0.2" + checksum: 2fc11503361b5fb4f14714c700c02a3f4c7c93e9acd6b87a29f62c522d90470f364d6161b03d1cc618b979f2ae02aed1106fd29d302695d8927e2fc8165ba8ee languageName: node linkType: hard @@ -2745,7 +2745,7 @@ __metadata: version: 0.0.0-use.local resolution: "@noir-lang/backend_barretenberg@portal:../noir/packages/backend_barretenberg::locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@aztec/bb.js": 0.16.0 + "@aztec/bb.js": 0.17.0 "@noir-lang/types": 0.22.0 fflate: ^0.8.0 languageName: node @@ -3103,12 +3103,12 @@ __metadata: linkType: hard "@types/eslint@npm:*": - version: 8.56.1 - resolution: "@types/eslint@npm:8.56.1" + version: 8.56.2 + resolution: "@types/eslint@npm:8.56.2" dependencies: "@types/estree": "*" "@types/json-schema": "*" - checksum: 1a4c7334c2f0cfead7b9d25c574c7b3d0b44242958703caa868ed38990a96b5d96477e6fceb7be54fbadd6fb61c97b778b9df58531ced3ec4b022d3e54254dc3 + checksum: 38e054971596f5c0413f66a62dc26b10e0a21ac46ceacb06fbf8cfb838d20820787209b17218b3916e4c23d990ff77cfdb482d655cac0e0d2b837d430fcc5db8 languageName: node linkType: hard @@ -3510,11 +3510,11 @@ __metadata: linkType: hard "@types/node@npm:*": - version: 20.10.8 - resolution: "@types/node@npm:20.10.8" + version: 20.11.0 + resolution: "@types/node@npm:20.11.0" dependencies: undici-types: ~5.26.4 - checksum: ce9b7ee545b3605f667be2ea900e38ab58d7b561192a7342443e5d7f61c44fd9d016eac48e95d3011f090ceea65a727e83a31d51fabdd9fc20ff9992edcbc682 + checksum: 1bd6890db7e0404d11c33d28f46f19f73256f0ba35d19f0ef2a0faba09f366f188915fb9338eebebcc472075c1c4941e17c7002786aa69afa44980737846b200 languageName: node linkType: hard @@ -6061,9 +6061,9 @@ __metadata: linkType: hard "electron-to-chromium@npm:^1.4.601": - version: 1.4.626 - resolution: "electron-to-chromium@npm:1.4.626" - checksum: 5ecfab63c6e9415e0bce577e53a95b2206ea3897d7f8026f2826d063f561be1fdcb6ded0f4b2ec40f00821ece34ca21f9c9357a80e5e343db4254bb9b94e65b0 + version: 1.4.628 + resolution: "electron-to-chromium@npm:1.4.628" + checksum: 113b475a3a869123f64ea9b7fc53b59a842c270882c9bc24318b63222fb721886a353fbecafde10d0df8389aff7d020b84e15565b65fe43c6f58c9e03aced1cd languageName: node linkType: hard From d64ca3f9a294f85df46be74bd6b0f17b3390835b Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 11 Jan 2024 09:55:09 +0000 Subject: [PATCH 07/50] update scripts --- aztec-up/bin/docker-compose.yml | 1 + yarn-project/aztec/Dockerfile | 2 +- yarn-project/end-to-end/scripts/docker-compose.yml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/aztec-up/bin/docker-compose.yml b/aztec-up/bin/docker-compose.yml index d99d9d27cf1..6716ca6415b 100644 --- a/aztec-up/bin/docker-compose.yml +++ b/aztec-up/bin/docker-compose.yml @@ -18,6 +18,7 @@ services: aztec: image: "aztecprotocol/aztec" + command: sandbox ports: - "${AZTEC_NODE_PORT:-8079}:${AZTEC_NODE_PORT:-8079}" - "${PXE_PORT:-8080}:${PXE_PORT:-8080}" diff --git a/yarn-project/aztec/Dockerfile b/yarn-project/aztec/Dockerfile index d36274c527c..2063bcc92ed 100644 --- a/yarn-project/aztec/Dockerfile +++ b/yarn-project/aztec/Dockerfile @@ -1,6 +1,6 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/aztec/dest/bin/index.js"] -EXPOSE 8079 8080 +EXPOSE 79 80 # The version has been updated in yarn-project-prod. # Adding COMMIT_TAG here to rebuild versioned image. diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 6a05e652777..8758c8a306c 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -14,6 +14,7 @@ services: sandbox: image: aztecprotocol/aztec:latest + command: sandbox environment: DEBUG: 'aztec:*' DEBUG_COLORS: 1 From d732330f17ede3406b852283c4e57b7507a3272d Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 11 Jan 2024 09:56:14 +0000 Subject: [PATCH 08/50] remove old sandbox bin --- yarn-project/aztec/src/bin/_index.ts | 245 --------------------------- 1 file changed, 245 deletions(-) delete mode 100644 yarn-project/aztec/src/bin/_index.ts diff --git a/yarn-project/aztec/src/bin/_index.ts b/yarn-project/aztec/src/bin/_index.ts deleted file mode 100644 index 75b8d311347..00000000000 --- a/yarn-project/aztec/src/bin/_index.ts +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env -S node --no-warnings -import { deployInitialTestAccounts } from '@aztec/accounts/testing'; -import { createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; -import { AccountManager, createAztecNodeClient } from '@aztec/aztec.js'; -import { NULL_KEY } from '@aztec/ethereum'; -import { init } from '@aztec/foundation/crypto'; -import { createStatusRouter, startHttpRpcServer } from '@aztec/foundation/json-rpc/server'; -import { createDebugLogger } from '@aztec/foundation/log'; -import { fileURLToPath } from '@aztec/foundation/url'; -import { NoirCommit, NoirTag } from '@aztec/noir-compiler/versions'; -import { BootstrapNode, getP2PConfigEnvVars } from '@aztec/p2p'; -import { GrumpkinScalar, PXEService, createPXERpcServer } from '@aztec/pxe'; - -import { lookup } from 'dns/promises'; -import { readFileSync } from 'fs'; -import http from 'http'; -import { dirname, resolve } from 'path'; -import { mnemonicToAccount } from 'viem/accounts'; - -import { setupFileDebugLog } from '../logging.js'; -import { MNEMONIC, createAztecNode, createAztecPXE, createSandbox, deployContractsToL1 } from '../sandbox.js'; -import { github, splash } from '../splash.js'; - -/** - * The mode in which the sandbox should be run. - */ -enum SandboxMode { - Sandbox = 'sandbox', - Node = 'node', - PXE = 'pxe', - P2PBootstrap = 'p2p-bootstrap', -} - -/** - * If we can successfully resolve 'host.docker.internal', then we are running in a container, and we should treat - * localhost as being host.docker.internal. - */ -const getLocalhost = () => - lookup('host.docker.internal') - .then(() => 'host.docker.internal') - .catch(() => 'localhost'); - -const LOCALHOST = await getLocalhost(); -const { - AZTEC_NODE_URL = `http://${LOCALHOST}:8079`, - AZTEC_NODE_PORT = 8079, - PXE_PORT = 8080, - MODE = 'sandbox', - TEST_ACCOUNTS = 'true', - DEPLOY_AZTEC_CONTRACTS = 'true', - API_PREFIX = '', -} = process.env; - -const logger = createDebugLogger(`aztec:${MODE}`); - -/** - * Creates the sandbox from provided config and deploys any initial L1 and L2 contracts - */ -async function createAndInitialiseSandbox(deployTestAccounts: boolean) { - const { aztecNodeConfig, node, pxe, stop } = await createSandbox(); - if (aztecNodeConfig.p2pEnabled) { - logger.info(`Not setting up test accounts as we are connecting to a network`); - return { - aztecNodeConfig, - pxe, - node, - stop, - accounts: [], - }; - } - let accounts; - if (deployTestAccounts) { - logger.info('Setting up test accounts...'); - accounts = await deployInitialTestAccounts(pxe); - } - return { - aztecNodeConfig, - pxe, - node, - stop, - accounts, - }; -} - -/** - * Create and start a new Aztec RPC HTTP Server - */ -async function main() { - const deployTestAccounts = TEST_ACCOUNTS === 'true'; - const deployAztecContracts = DEPLOY_AZTEC_CONTRACTS === 'true'; - - const mode = MODE as SandboxMode; - - const installSignalHandlers = (cb?: () => Promise) => { - const shutdown = async () => { - logger.info('Shutting down...'); - if (cb) { - await cb(); - } - process.exit(0); - }; - process.removeAllListeners('SIGINT'); - process.removeAllListeners('SIGTERM'); - process.once('SIGINT', shutdown); - process.once('SIGTERM', shutdown); - }; - - installSignalHandlers(); - - // Init crypto (bb.js). - await init(); - - const logStrings = []; - - const logPath = setupFileDebugLog(); - logger.info(`Debug logs will be written to ${logPath}`); - - // Get Sandbox version - const packageJsonPath = resolve(dirname(fileURLToPath(import.meta.url)), '../../package.json'); - const version = JSON.parse(readFileSync(packageJsonPath).toString()).version; - - // Code path for starting Sandbox - if (mode === SandboxMode.Sandbox) { - logger.info(`Setting up Aztec Sandbox v${version} (noir ${NoirCommit} ${NoirTag}), please stand by...`); - - const { pxe, node, stop, accounts } = await createAndInitialiseSandbox(deployTestAccounts); - - // Create shutdown cleanup function - installSignalHandlers(stop); - - // Start Node and PXE JSON-RPC servers - startHttpRpcServer(node, createAztecNodeRpcServer, AZTEC_NODE_PORT); - logger.info(`Aztec Node JSON-RPC Server listening on port ${AZTEC_NODE_PORT}`); - startHttpRpcServer(pxe, createPXERpcServer, PXE_PORT); - logger.info(`PXE JSON-RPC Server listening on port ${PXE_PORT}`); - - // Log initial accounts details - if (accounts?.length) { - const accountLogStrings = await createAccountLogs(accounts, pxe); - logStrings.push(...accountLogStrings); - } - logStrings.push(`Aztec Sandbox v${version} (noir ${NoirCommit}) is now ready for use!`); - } else if (mode === SandboxMode.Node) { - // Code path for starting Node only - const nodeConfig = getNodeConfigEnvVars(); - const hdAccount = mnemonicToAccount(MNEMONIC); - - // Deploy L1 Aztec Contracts if needed - if (deployAztecContracts) { - await deployContractsToL1(nodeConfig, hdAccount); - if (nodeConfig.publisherPrivateKey === NULL_KEY) { - const privKey = hdAccount.getHdKey().privateKey; - nodeConfig.publisherPrivateKey = `0x${Buffer.from(privKey!).toString('hex')}`; - } - } - - const node = await createAztecNode(nodeConfig); - installSignalHandlers(node.stop); - - const port = process.env.AZTEC_NODE_PORT || 8080; // Use standard 8080 when no PXE is running - const nodeRpcServer = createAztecNodeRpcServer(node); - const app = nodeRpcServer.getApp(API_PREFIX); - - // Add a /status endpoint - const statusRouter = createStatusRouter(API_PREFIX); - app.use(statusRouter.routes()); - app.use(statusRouter.allowedMethods()); - - // Start Node JSON-RPC server - const httpServer = http.createServer(app.callback()); - httpServer.listen(port); - - logStrings.push(`Aztec Node v${version} (noir ${NoirCommit}) is now ready for use in port ${port}!`); - } else if (mode === SandboxMode.PXE) { - // Code path for starting PXE only - - // Create a Node client to connect to the PXE - const node = createAztecNodeClient(AZTEC_NODE_URL); - - const pxe = await createAztecPXE(node); - installSignalHandlers(pxe.stop); - - // Start PXE JSON-RPC server - startHttpRpcServer(pxe, createPXERpcServer, PXE_PORT); - - if (deployTestAccounts) { - logger.info('Setting up test accounts...'); - const accounts = await deployInitialTestAccounts(pxe); - const accountLogStrings = await createAccountLogs(accounts, pxe); - logStrings.push(...accountLogStrings); - } - - logStrings.push(`PXE v${version} (noir ${NoirCommit}) is now ready for use in port ${PXE_PORT}!`); - } else if (mode === SandboxMode.P2PBootstrap) { - // Code path for starting a P2P bootstrap node - const config = getP2PConfigEnvVars(); - const bootstrapNode = new BootstrapNode(logger); - await bootstrapNode.start(config); - installSignalHandlers(bootstrapNode.stop); - logStrings.push( - `Bootstrap P2P node is now ready for use. Listening on: ${config.tcpListenIp}:${config.tcpListenPort}.`, - ); - } - - // Log startup details - logger.info(`${splash}\n${github}\n\n`.concat(...logStrings)); -} - -/** - * Creates logs for the initial accounts - * @param accounts - The initial accounts - * @param pxe - A PXE instance to get the registered accounts - * @returns A string array containing the initial accounts details - */ -async function createAccountLogs( - accounts: { - /** - * The account object - */ - account: AccountManager; - /** - * The private key of the account - */ - privateKey: GrumpkinScalar; - }[], - pxe: PXEService, -) { - const registeredAccounts = await pxe.getRegisteredAccounts(); - const accountLogStrings = [`Initial Accounts:\n\n`]; - for (const account of accounts) { - const completeAddress = account.account.getCompleteAddress(); - if (registeredAccounts.find(a => a.equals(completeAddress))) { - accountLogStrings.push(` Address: ${completeAddress.address.toString()}\n`); - accountLogStrings.push(` Partial Address: ${completeAddress.partialAddress.toString()}\n`); - accountLogStrings.push(` Private Key: ${account.privateKey.toString()}\n`); - accountLogStrings.push(` Public Key: ${completeAddress.publicKey.toString()}\n\n`); - } - } - return accountLogStrings; -} - -main().catch(err => { - logger.error(err); - process.exit(1); -}); From a1d1c14a4b98c861a9dd7c178ba1f1c523708b0e Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 11 Jan 2024 09:59:17 +0000 Subject: [PATCH 09/50] add 'aztec cli' call --- aztec-up/bin/aztec | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/aztec-up/bin/aztec b/aztec-up/bin/aztec index 31e43f651be..16402c7be85 100755 --- a/aztec-up/bin/aztec +++ b/aztec-up/bin/aztec @@ -1,4 +1,10 @@ #!/usr/bin/env bash set -euo pipefail -$(dirname $0)/.aztec-run aztecprotocol/aztec $@ +# Call cli image if used with `aztec cli ...args` +if [ "$1" == "cli" ]; then + shift + $(dirname $0)/.aztec-run aztecprotocol/cli $@ +else + $(dirname $0)/.aztec-run aztecprotocol/aztec $@ +fi From d685c0445c48899330ef6e75ea60acb4335bbbcf Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 11 Jan 2024 14:05:07 +0000 Subject: [PATCH 10/50] fix compose files --- boxes/docker-compose.yml | 1 + yarn-project/end-to-end/scripts/docker-compose-browser.yml | 1 + yarn-project/end-to-end/scripts/docker-compose-p2p.yml | 2 +- yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/boxes/docker-compose.yml b/boxes/docker-compose.yml index e731d25c01e..3aa48f09860 100644 --- a/boxes/docker-compose.yml +++ b/boxes/docker-compose.yml @@ -6,6 +6,7 @@ services: aztec: image: aztecprotocol/aztec + command: sandbox environment: ETHEREUM_HOST: http://ethereum:8545 CHAIN_ID: 31337 diff --git a/yarn-project/end-to-end/scripts/docker-compose-browser.yml b/yarn-project/end-to-end/scripts/docker-compose-browser.yml index f859de7de39..c812613848b 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-browser.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-browser.yml @@ -14,6 +14,7 @@ services: sandbox: image: aztecprotocol/aztec:latest + command: sandbox environment: DEBUG: 'aztec:*' DEBUG_COLORS: 1 diff --git a/yarn-project/end-to-end/scripts/docker-compose-p2p.yml b/yarn-project/end-to-end/scripts/docker-compose-p2p.yml index 4088526269f..b7f9073ba7b 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-p2p.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-p2p.yml @@ -8,11 +8,11 @@ services: p2p-bootstrap: image: aztecprotocol/aztec:latest + command: 'start --p2p-bootstrap' ports: - '40400:40400' command: 'start' environment: - MODE: 'p2p-bootstrap' DEBUG: 'aztec:*' DEBUG_COLORS: 1 P2P_TCP_LISTEN_PORT: 40400 diff --git a/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml b/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml index 236b889dab6..9ea2293cb24 100644 --- a/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml +++ b/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml @@ -2,10 +2,10 @@ version: '3' services: p2p-bootstrap: image: 278380418400.dkr.ecr.eu-west-2.amazonaws.com/aztec:latest + command: 'start --p2p-bootstrap' ports: - '40400:40400' environment: - MODE: 'p2p-bootstrap' DEBUG: 'aztec:*' P2P_TCP_LISTEN_PORT: 40400 PEER_ID: '0a260024080112205ea53185db2e52dae74d0d4d6cadc494174810d0a713cd09b0ac517c38bc781e1224080112205ea53185db2e52dae74d0d4d6cadc494174810d0a713cd09b0ac517c38bc781e1a44080112402df8b977f356c6e34fa021c9647973234dff4df706c185794405aafb556723cf5ea53185db2e52dae74d0d4d6cadc494174810d0a713cd09b0ac517c38bc781e' From 48edee52e2cbcc663cdf781567c600b3f76a6e9d Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 12 Jan 2024 12:37:52 +0000 Subject: [PATCH 11/50] fix ports --- yarn-project/aztec/package.json | 1 - yarn-project/aztec/src/cli/cli.ts | 20 +++++++++---------- .../end-to-end/scripts/docker-compose.yml | 2 ++ 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index e9cdbb2da1f..b401c02e5b3 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -16,7 +16,6 @@ "scripts": { "build": "yarn clean && tsc -b", "start": "node --no-warnings ./dest/bin", - "start:cli": "node --no-warnings ./dest/bin/cli.js", "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index 44dfd1d1dfe..5a80e7bc9ad 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -20,8 +20,8 @@ import { github, splash } from '../splash.js'; import { cliTexts } from './texts.js'; const { - AZTEC_NODE_PORT = '80', - PXE_PORT = '79', + AZTEC_NODE_PORT = '8079', + PXE_PORT = '8080', ARCHIVER_PORT = '80', AZTEC_NODE_URL, DEPLOY_AZTEC_CONTRACTS, @@ -93,9 +93,9 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { program .command('sandbox') .description('Starts Aztec sandbox.') - .option('-p, --port ', 'Port to run sandbox on.', AZTEC_NODE_PORT) - .option('-pp, --pxe-port ', 'Port to run PXE on (optional).') - .option('-s, --skip-test-accounts', 'DO NOT deploy test accounts.', true) + .option('-np, --node-port ', 'Port to run Aztec Node on.', AZTEC_NODE_PORT) + .option('-pp, --pxe-port ', 'Port to run PXE on (optional).', PXE_PORT) + .option('-s, --skip-test-accounts', 'DO NOT deploy test accounts.', false) // .option() .action(async options => { userLog(`${splash}\n${github}\n\n`); @@ -116,12 +116,10 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { } // Start Node and PXE JSON-RPC servers - startHttpRpcServer(node, createAztecNodeRpcServer, options.port); - userLog(`Aztec Node JSON-RPC Server listening on port ${options.port}`); - if (options.pxePort) { - startHttpRpcServer(pxe, createPXERpcServer, PXE_PORT); - userLog(`PXE JSON-RPC Server listening on port ${PXE_PORT}`); - } + startHttpRpcServer(node, createAztecNodeRpcServer, options.nodePort); + userLog(`Aztec Node JSON-RPC Server listening on nodePort ${options.nodePort}`); + startHttpRpcServer(pxe, createPXERpcServer, options.pxePort); + userLog(`PXE JSON-RPC Server listening on port ${options.pxePort}`); }); // Start Aztec modules with options diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 8758c8a306c..8b00ebff566 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -17,6 +17,7 @@ services: command: sandbox environment: DEBUG: 'aztec:*' + PXE_PORT: 8080 DEBUG_COLORS: 1 ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 @@ -28,6 +29,7 @@ services: ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 ports: - '8080:8080' + - '8079:8079' end-to-end: image: aztecprotocol/end-to-end:latest From 2950770dff4582f9ea94d491b31bac2368d26e0f Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 12 Jan 2024 14:51:39 +0000 Subject: [PATCH 12/50] fix typo --- yarn-project/aztec.js/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/aztec.js/src/index.ts b/yarn-project/aztec.js/src/index.ts index 67b4e0b3340..516f6c9245c 100644 --- a/yarn-project/aztec.js/src/index.ts +++ b/yarn-project/aztec.js/src/index.ts @@ -107,7 +107,7 @@ export { merkleTreeIds, mockTx, } from '@aztec/circuit-types'; -export { NodeInfro } from '@aztec/types/interfaces'; +export { NodeInfo } from '@aztec/types/interfaces'; // TODO: These kinds of things have no place on our public api. // External devs will almost certainly have their own methods of doing these things. From 72ade948554c51866ed2e61f0e809246e69269c7 Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 12 Jan 2024 15:23:39 +0000 Subject: [PATCH 13/50] fix more imports --- yarn-project/accounts/src/defaults/account_contract.ts | 3 ++- yarn-project/accounts/src/defaults/account_interface.ts | 3 ++- yarn-project/cli/src/client.test.ts | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/yarn-project/accounts/src/defaults/account_contract.ts b/yarn-project/accounts/src/defaults/account_contract.ts index 1247248995c..2f896713be4 100644 --- a/yarn-project/accounts/src/defaults/account_contract.ts +++ b/yarn-project/accounts/src/defaults/account_contract.ts @@ -1,5 +1,6 @@ +import { NodeInfo } from '@aztec/aztec.js'; import { AccountContract, AccountInterface, AuthWitnessProvider } from '@aztec/aztec.js/account'; -import { CompleteAddress, NodeInfo } from '@aztec/circuit-types'; +import { CompleteAddress } from '@aztec/circuit-types'; import { ContractArtifact } from '@aztec/foundation/abi'; import { DefaultAccountInterface } from '../defaults/account_interface.js'; diff --git a/yarn-project/accounts/src/defaults/account_interface.ts b/yarn-project/accounts/src/defaults/account_interface.ts index f4bbd465cf2..08cb3c58416 100644 --- a/yarn-project/accounts/src/defaults/account_interface.ts +++ b/yarn-project/accounts/src/defaults/account_interface.ts @@ -1,5 +1,6 @@ +import { NodeInfo } from '@aztec/aztec.js'; import { AccountInterface, AuthWitnessProvider, EntrypointInterface } from '@aztec/aztec.js/account'; -import { AuthWitness, FunctionCall, NodeInfo, TxExecutionRequest } from '@aztec/circuit-types'; +import { AuthWitness, FunctionCall, TxExecutionRequest } from '@aztec/circuit-types'; import { CompleteAddress, Fr } from '@aztec/circuits.js'; import { DefaultAccountEntrypoint } from './account_entrypoint.js'; diff --git a/yarn-project/cli/src/client.test.ts b/yarn-project/cli/src/client.test.ts index 11823a90b25..bcace613eeb 100644 --- a/yarn-project/cli/src/client.test.ts +++ b/yarn-project/cli/src/client.test.ts @@ -1,4 +1,5 @@ -import { NodeInfo, PXE } from '@aztec/circuit-types'; +import { NodeInfo } from '@aztec/aztec.js'; +import { PXE } from '@aztec/circuit-types'; import { MockProxy, mock } from 'jest-mock-extended'; From 7056584ef0641f5dfc9427cebf2712158a30b4e4 Mon Sep 17 00:00:00 2001 From: spypsy Date: Mon, 15 Jan 2024 11:58:55 +0000 Subject: [PATCH 14/50] fix build --- yarn-project/aztec.js/src/index.ts | 2 -- yarn-project/aztec.js/src/rpc_clients/aztec_node_client.ts | 2 +- yarn-project/types/package.json | 2 +- yarn-project/yarn.lock | 3 --- 4 files changed, 2 insertions(+), 7 deletions(-) diff --git a/yarn-project/aztec.js/src/index.ts b/yarn-project/aztec.js/src/index.ts index b2b13877eaf..516f6c9245c 100644 --- a/yarn-project/aztec.js/src/index.ts +++ b/yarn-project/aztec.js/src/index.ts @@ -109,8 +109,6 @@ export { } from '@aztec/circuit-types'; export { NodeInfo } from '@aztec/types/interfaces'; -export { NodeInfo } from '@aztec/types/interfaces'; - // TODO: These kinds of things have no place on our public api. // External devs will almost certainly have their own methods of doing these things. // If we want to use them in our own "aztec.js consuming code", import them from foundation as needed. diff --git a/yarn-project/aztec.js/src/rpc_clients/aztec_node_client.ts b/yarn-project/aztec.js/src/rpc_clients/aztec_node_client.ts index c39709324d8..7520eaee226 100644 --- a/yarn-project/aztec.js/src/rpc_clients/aztec_node_client.ts +++ b/yarn-project/aztec.js/src/rpc_clients/aztec_node_client.ts @@ -8,7 +8,6 @@ import { L2BlockL2Logs, L2Tx, LogId, - SiblingPath, Tx, TxHash, } from '@aztec/circuit-types'; @@ -17,6 +16,7 @@ import { AztecAddress } from '@aztec/foundation/aztec-address'; import { EthAddress } from '@aztec/foundation/eth-address'; import { Fr } from '@aztec/foundation/fields'; import { createJsonRpcClient, defaultFetch } from '@aztec/foundation/json-rpc/client'; +import { SiblingPath } from '@aztec/types/membership'; /** * Creates a JSON-RPC client to remotely talk to an Aztec Node. diff --git a/yarn-project/types/package.json b/yarn-project/types/package.json index 2ead60a11b0..2ebd3ab048d 100644 --- a/yarn-project/types/package.json +++ b/yarn-project/types/package.json @@ -62,4 +62,4 @@ "engines": { "node": ">=18" } -} \ No newline at end of file +} diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index eaaa482d175..dfe0b64f3cb 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -873,10 +873,7 @@ __metadata: resolution: "@aztec/types@workspace:types" dependencies: "@aztec/ethereum": "workspace:^" -<<<<<<< HEAD -======= "@aztec/foundation": "workspace:^" ->>>>>>> master "@jest/globals": ^29.5.0 "@types/debug": ^4.1.7 "@types/detect-node": ^2.0.0 From 755cb6059232c12d47c32ed4eb4302121e9e4ba6 Mon Sep 17 00:00:00 2001 From: spypsy Date: Mon, 15 Jan 2024 12:10:58 +0000 Subject: [PATCH 15/50] fix imports --- yarn-project/accounts/src/defaults/account_contract.ts | 1 - yarn-project/accounts/src/defaults/account_interface.ts | 1 - yarn-project/archiver/src/archiver/archiver.ts | 4 +++- yarn-project/aztec-node/src/aztec-node/server.ts | 3 +-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/yarn-project/accounts/src/defaults/account_contract.ts b/yarn-project/accounts/src/defaults/account_contract.ts index 4141beead8f..eed3e159e6c 100644 --- a/yarn-project/accounts/src/defaults/account_contract.ts +++ b/yarn-project/accounts/src/defaults/account_contract.ts @@ -1,4 +1,3 @@ -import { NodeInfo } from '@aztec/aztec.js'; import { AccountContract, AccountInterface, AuthWitnessProvider } from '@aztec/aztec.js/account'; import { CompleteAddress } from '@aztec/circuit-types'; import { ContractArtifact } from '@aztec/foundation/abi'; diff --git a/yarn-project/accounts/src/defaults/account_interface.ts b/yarn-project/accounts/src/defaults/account_interface.ts index 1fdb549b30b..d04ab0630e0 100644 --- a/yarn-project/accounts/src/defaults/account_interface.ts +++ b/yarn-project/accounts/src/defaults/account_interface.ts @@ -1,4 +1,3 @@ -import { NodeInfo } from '@aztec/aztec.js'; import { AccountInterface, AuthWitnessProvider, EntrypointInterface } from '@aztec/aztec.js/account'; import { AuthWitness, FunctionCall, TxExecutionRequest } from '@aztec/circuit-types'; import { CompleteAddress, Fr } from '@aztec/circuits.js'; diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index 55bc7f09580..566dcddc9fa 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -36,12 +36,14 @@ import { retrieveNewPendingL1ToL2Messages, } from './data_retrieval.js'; +export type ArchiveSource = L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource; + /** * Pulls L2 blocks in a non-blocking manner and provides interface for their retrieval. * Responsible for handling robust L1 polling so that other components do not need to * concern themselves with it. */ -export class Archiver implements L2BlockSource, L2LogsSource, ContractDataSource, L1ToL2MessageSource { +export class Archiver implements ArchiveSource { /** * A promise in which we will be continually fetching new L2 blocks. */ diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 0be7dd02627..0fc364ccd80 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -1,7 +1,6 @@ -import { Archiver, LMDBArchiverStore } from '@aztec/archiver'; +import { ArchiveSource, Archiver, ArchiverDataStore, LMDBArchiverStore } from '@aztec/archiver'; import { createArchiverClient } from '@aztec/aztec.js'; import { - ArchiveSource, AztecNode, ContractData, ContractDataSource, From 1db41ccbaa61d55b72ea05d8eb27b220bcda2bf4 Mon Sep 17 00:00:00 2001 From: spypsy Date: Mon, 15 Jan 2024 12:41:36 +0000 Subject: [PATCH 16/50] remove openDB fn from kv-store --- .../src/aztec-node/db.ts} | 13 +--- .../aztec-node/src/aztec-node/server.ts | 3 +- yarn-project/aztec-node/tsconfig.json | 6 +- yarn-project/aztec/package.json | 1 - yarn-project/aztec/src/cli/cli.ts | 4 +- yarn-project/aztec/src/cli/util.ts | 78 +++++++++++++++++++ yarn-project/aztec/tsconfig.json | 3 - yarn-project/kv-store/package.json | 5 +- yarn-project/kv-store/src/index.ts | 1 - yarn-project/pxe/package.json | 1 - yarn-project/pxe/tsconfig.json | 3 - yarn-project/yarn.lock | 5 -- 12 files changed, 88 insertions(+), 35 deletions(-) rename yarn-project/{kv-store/src/open_db.ts => aztec-node/src/aztec-node/db.ts} (89%) create mode 100644 yarn-project/aztec/src/cli/util.ts diff --git a/yarn-project/kv-store/src/open_db.ts b/yarn-project/aztec-node/src/aztec-node/db.ts similarity index 89% rename from yarn-project/kv-store/src/open_db.ts rename to yarn-project/aztec-node/src/aztec-node/db.ts index 07894f887c5..9b5be428781 100644 --- a/yarn-project/kv-store/src/open_db.ts +++ b/yarn-project/aztec-node/src/aztec-node/db.ts @@ -1,4 +1,3 @@ -import { EthAddress } from '@aztec/foundation/eth-address'; import { LogFn } from '@aztec/foundation/log'; import { LevelDown, default as leveldown } from 'leveldown'; @@ -8,6 +7,8 @@ import { MemDown, default as memdown } from 'memdown'; import { mkdir } from 'node:fs/promises'; import { join } from 'node:path'; +import { AztecNodeConfig } from './config.js'; + export const createMemDown = () => (memdown as any)() as MemDown; export const createLevelDown = (path: string) => (leveldown as any)(path) as LevelDown; @@ -32,15 +33,7 @@ type NodeMetadata = { * @returns The database for the aztec node. */ export async function openDb( - config: { - /** The directory to store the database in. If not specified, a temporary database is used. */ - dataDirectory?: string; - /** The addresses of the Aztec contracts on L1. */ - l1Contracts: { - /** The address of the Aztec rollup contract on L1. */ - rollupAddress: EthAddress; - }; - }, + config: AztecNodeConfig, log: LogFn, ): Promise<[nodeDb: RootDatabase, worldStateDb: LevelUp]> { const nodeMetadata: NodeMetadata = { diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 0fc364ccd80..0e86055fee1 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -40,7 +40,7 @@ import { computeGlobalsHash, computePublicDataTreeLeafSlot } from '@aztec/circui import { L1ContractAddresses, createEthereumChain } from '@aztec/ethereum'; import { AztecAddress } from '@aztec/foundation/aztec-address'; import { createDebugLogger } from '@aztec/foundation/log'; -import { AztecLmdbStore, openDb } from '@aztec/kv-store'; +import { AztecLmdbStore } from '@aztec/kv-store'; import { AztecKVTxPool, P2P, createP2PClient } from '@aztec/p2p'; import { GlobalVariableBuilder, @@ -60,6 +60,7 @@ import { import { LevelUp } from 'levelup'; import { AztecNodeConfig } from './config.js'; +import { openDb } from './db.js'; /** * The aztec node. diff --git a/yarn-project/aztec-node/tsconfig.json b/yarn-project/aztec-node/tsconfig.json index e19c468d762..23f2a6a349f 100644 --- a/yarn-project/aztec-node/tsconfig.json +++ b/yarn-project/aztec-node/tsconfig.json @@ -46,7 +46,5 @@ "path": "../world-state" } ], - "include": [ - "src" - ] -} \ No newline at end of file + "include": ["src"] +} diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index b7de5737e9e..c7e55554cf2 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -35,7 +35,6 @@ "@aztec/circuits.js": "workspace:^", "@aztec/ethereum": "workspace:^", "@aztec/foundation": "workspace:^", - "@aztec/kv-store": "workspace:^", "@aztec/l1-artifacts": "workspace:^", "@aztec/noir-compiler": "workspace:^", "@aztec/noir-contracts": "workspace:^", diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index 5a80e7bc9ad..fef43b7f84a 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -5,7 +5,6 @@ import { AccountManager, GrumpkinScalar, createAztecNodeClient, fileURLToPath } import { NULL_KEY } from '@aztec/ethereum'; import { startHttpRpcServer } from '@aztec/foundation/json-rpc/server'; import { DebugLogger, LogFn } from '@aztec/foundation/log'; -import { openDb } from '@aztec/kv-store'; import { BootstrapNode, P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; import { PXEService, PXEServiceConfig, createPXERpcServer, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; @@ -18,6 +17,7 @@ import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; import { MNEMONIC, createAztecNode, createAztecPXE, createSandbox, deployContractsToL1 } from '../sandbox.js'; import { github, splash } from '../splash.js'; import { cliTexts } from './texts.js'; +import { openDb } from './util.js'; const { AZTEC_NODE_PORT = '8079', @@ -231,7 +231,7 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { // merge env vars and cli options const archiverConfig = pick({ ...archiverConfigEnvVars, ...archiverCliOptions }); - const [nodeDb] = await openDb(archiverConfig, debugLogger); + const nodeDb = await openDb(archiverConfig, debugLogger); const archiverStore = new LMDBArchiverStore(nodeDb, archiverConfig.maxLogs); const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, true); diff --git a/yarn-project/aztec/src/cli/util.ts b/yarn-project/aztec/src/cli/util.ts new file mode 100644 index 00000000000..49b7b646d59 --- /dev/null +++ b/yarn-project/aztec/src/cli/util.ts @@ -0,0 +1,78 @@ +import { ArchiverConfig } from '@aztec/archiver'; +import { LogFn } from '@aztec/foundation/log'; + +import { LevelDown, default as leveldown } from 'leveldown'; +import { LevelUp, default as levelup } from 'levelup'; +import { RootDatabase, open } from 'lmdb'; +import { MemDown, default as memdown } from 'memdown'; +import { mkdir } from 'node:fs/promises'; +import { join } from 'node:path'; + +export const createMemDown = () => (memdown as any)() as MemDown; +export const createLevelDown = (path: string) => (leveldown as any)(path) as LevelDown; + +const DB_SUBDIR = 'aztec-node-db'; +const WORLD_STATE_SUBDIR = 'aztec-world-state-db'; +const NODE_METADATA_KEY = '@@aztec_node_metadata'; + +/** + * The metadata for an aztec node. + */ +type NodeMetadata = { + /** + * The address of the rollup contract on L1 + */ + rollupContractAddress: string; +}; + +/** + * Opens the database for the aztec node. If a data directory is specified, then this attempts to create it. + * @param config - The configuration to be used by the aztec node. + * @throws If `config.dataDirectory` is set and the directory cannot be created. + * @returns The database for the aztec node. + */ +export async function openDb(config: ArchiverConfig, log: LogFn): Promise { + const nodeMetadata: NodeMetadata = { + rollupContractAddress: config.l1Contracts.rollupAddress.toString(), + }; + + let nodeDb: RootDatabase; + let worldStateDb: LevelUp; + + if (config.dataDirectory) { + const nodeDir = join(config.dataDirectory, DB_SUBDIR); + // this throws if we don't have permissions to create the directory + await mkdir(nodeDir, { recursive: true }); + + log(`Opening aztec-node database at ${nodeDir}`); + nodeDb = open(nodeDir, {}); + } else { + log('Opening temporary databases'); + // not passing a path will use a temp file that gets deleted when the process exits + nodeDb = open({}); + worldStateDb = levelup(createMemDown()); + } + + await checkNodeMetadataAndClear(nodeDb, nodeMetadata, log); + return nodeDb; +} + +/** + * Checks the node metadata and clears the database if the rollup contract address has changed. + * @param nodeDb - The database for the aztec node. + * @param nodeMetadata - The metadata for the aztec node. + */ +async function checkNodeMetadataAndClear(nodeDb: RootDatabase, nodeMetadata: NodeMetadata, log: LogFn): Promise { + const metadataDB = nodeDb.openDB('metadata', {}); + try { + const existing = metadataDB.get(NODE_METADATA_KEY); + // if the rollup addresses are different, wipe the local database and start over + if (!existing || existing.rollupContractAddress !== nodeMetadata.rollupContractAddress) { + log('Rollup contract address has changed, clearing databases'); + await nodeDb.clearAsync(); + } + await metadataDB.put(NODE_METADATA_KEY, nodeMetadata); + } finally { + await metadataDB.close(); + } +} diff --git a/yarn-project/aztec/tsconfig.json b/yarn-project/aztec/tsconfig.json index a27fb82841d..4844bd37232 100644 --- a/yarn-project/aztec/tsconfig.json +++ b/yarn-project/aztec/tsconfig.json @@ -30,9 +30,6 @@ { "path": "../foundation" }, - { - "path": "../kv-store" - }, { "path": "../l1-artifacts" }, diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json index 94bf287e332..8aea043378d 100644 --- a/yarn-project/kv-store/package.json +++ b/yarn-project/kv-store/package.json @@ -26,10 +26,7 @@ }, "dependencies": { "@aztec/foundation": "workspace:^", - "leveldown": "^6.1.1", - "levelup": "^5.1.1", - "lmdb": "^2.9.1", - "memdown": "^6.1.1" + "lmdb": "^2.9.1" }, "devDependencies": { "@jest/globals": "^29.5.0", diff --git a/yarn-project/kv-store/src/index.ts b/yarn-project/kv-store/src/index.ts index c08d9ad3289..2a71333f9e6 100644 --- a/yarn-project/kv-store/src/index.ts +++ b/yarn-project/kv-store/src/index.ts @@ -3,4 +3,3 @@ export * from './interfaces/map.js'; export * from './interfaces/singleton.js'; export * from './interfaces/store.js'; export * from './lmdb/store.js'; -export * from './open_db.js'; diff --git a/yarn-project/pxe/package.json b/yarn-project/pxe/package.json index 9292143cf18..223b5f7abd4 100644 --- a/yarn-project/pxe/package.json +++ b/yarn-project/pxe/package.json @@ -34,7 +34,6 @@ }, "dependencies": { "@aztec/acir-simulator": "workspace:^", - "@aztec/aztec.js": "workspace:^", "@aztec/circuit-types": "workspace:^", "@aztec/circuits.js": "workspace:^", "@aztec/ethereum": "workspace:^", diff --git a/yarn-project/pxe/tsconfig.json b/yarn-project/pxe/tsconfig.json index ed17ca7e4be..f78f8c2e76a 100644 --- a/yarn-project/pxe/tsconfig.json +++ b/yarn-project/pxe/tsconfig.json @@ -9,9 +9,6 @@ { "path": "../acir-simulator" }, - { - "path": "../aztec.js" - }, { "path": "../circuit-types" }, diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index dfe0b64f3cb..79eac44ef2a 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -265,7 +265,6 @@ __metadata: "@aztec/circuits.js": "workspace:^" "@aztec/ethereum": "workspace:^" "@aztec/foundation": "workspace:^" - "@aztec/kv-store": "workspace:^" "@aztec/l1-artifacts": "workspace:^" "@aztec/noir-compiler": "workspace:^" "@aztec/noir-contracts": "workspace:^" @@ -572,10 +571,7 @@ __metadata: "@types/node": ^18.7.23 jest: ^29.5.0 jest-mock-extended: ^3.0.3 - leveldown: ^6.1.1 - levelup: ^5.1.1 lmdb: ^2.9.1 - memdown: ^6.1.1 ts-jest: ^29.1.0 ts-node: ^10.9.1 typescript: ^5.0.4 @@ -771,7 +767,6 @@ __metadata: resolution: "@aztec/pxe@workspace:pxe" dependencies: "@aztec/acir-simulator": "workspace:^" - "@aztec/aztec.js": "workspace:^" "@aztec/circuit-types": "workspace:^" "@aztec/circuits.js": "workspace:^" "@aztec/ethereum": "workspace:^" From 8d4cf647e73ad3b60d5ecf600e701c07600dd0d3 Mon Sep 17 00:00:00 2001 From: spypsy Date: Mon, 15 Jan 2024 14:09:29 +0000 Subject: [PATCH 17/50] undo formatting --- yarn-project/tsconfig.json | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/yarn-project/tsconfig.json b/yarn-project/tsconfig.json index 2f4204982f8..f648e5691f1 100644 --- a/yarn-project/tsconfig.json +++ b/yarn-project/tsconfig.json @@ -1,11 +1,7 @@ { "compilerOptions": { "target": "es2020", - "lib": [ - "dom", - "esnext", - "es2017.object" - ], + "lib": ["dom", "esnext", "es2017.object"], "module": "NodeNext", "moduleResolution": "NodeNext", "strict": true, @@ -49,7 +45,5 @@ { "path": "world-state/tsconfig.json" }, { "path": "scripts/tsconfig.json" } ], - "files": [ - "./@types/jest/index.d.ts" - ] + "files": ["./@types/jest/index.d.ts"] } \ No newline at end of file From 1531044ec9cfcd91164f74314690a50369e8d053 Mon Sep 17 00:00:00 2001 From: spypsy Date: Mon, 15 Jan 2024 15:26:52 +0000 Subject: [PATCH 18/50] move rpc clients --- .../archiver/src/archiver/archiver.ts | 3 + yarn-project/archiver/src/archiver/index.ts | 1 - yarn-project/archiver/src/index.ts | 1 + .../src/rpc}/archiver_client.ts | 0 .../archiver_server.ts} | 2 +- yarn-project/archiver/src/rpc/index.ts | 2 + .../aztec-node/src/aztec-node/server.ts | 3 +- yarn-project/aztec.js/src/index.ts | 3 +- .../src/rpc_clients/aztec_node_client.ts | 50 ----------------- .../aztec.js/src/rpc_clients/index.ts | 2 - yarn-project/aztec/package.json | 3 + yarn-project/aztec/src/cli/cli.ts | 3 +- yarn-project/aztec/src/cli/util.ts | 56 +++++++++---------- yarn-project/end-to-end/package.json | 1 - yarn-project/end-to-end/tsconfig.json | 3 - yarn-project/yarn.lock | 8 ++- 16 files changed, 48 insertions(+), 93 deletions(-) rename yarn-project/{aztec.js/src/rpc_clients => archiver/src/rpc}/archiver_client.ts (100%) rename yarn-project/archiver/src/{archiver/archiver_http_server.ts => rpc/archiver_server.ts} (95%) create mode 100644 yarn-project/archiver/src/rpc/index.ts delete mode 100644 yarn-project/aztec.js/src/rpc_clients/aztec_node_client.ts diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index 566dcddc9fa..6d71a352a45 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -36,6 +36,9 @@ import { retrieveNewPendingL1ToL2Messages, } from './data_retrieval.js'; +/** + * Helper interface to combine all sources this archiver implementation provides. + */ export type ArchiveSource = L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource; /** diff --git a/yarn-project/archiver/src/archiver/index.ts b/yarn-project/archiver/src/archiver/index.ts index 036e32f5a68..0ef2b0025c8 100644 --- a/yarn-project/archiver/src/archiver/index.ts +++ b/yarn-project/archiver/src/archiver/index.ts @@ -3,4 +3,3 @@ export * from './config.js'; export { MemoryArchiverStore } from './memory_archiver_store/memory_archiver_store.js'; export { LMDBArchiverStore } from './lmdb_archiver_store.js'; export { ArchiverDataStore } from './archiver_store.js'; -export * from './archiver_http_server.js'; diff --git a/yarn-project/archiver/src/index.ts b/yarn-project/archiver/src/index.ts index c2f0d128be8..e47ee3fcc37 100644 --- a/yarn-project/archiver/src/index.ts +++ b/yarn-project/archiver/src/index.ts @@ -8,6 +8,7 @@ import { Archiver, getConfigEnvVars } from './archiver/index.js'; import { MemoryArchiverStore } from './archiver/memory_archiver_store/memory_archiver_store.js'; export * from './archiver/index.js'; +export * from './rpc/index.js'; const log = createDebugLogger('aztec:archiver'); diff --git a/yarn-project/aztec.js/src/rpc_clients/archiver_client.ts b/yarn-project/archiver/src/rpc/archiver_client.ts similarity index 100% rename from yarn-project/aztec.js/src/rpc_clients/archiver_client.ts rename to yarn-project/archiver/src/rpc/archiver_client.ts diff --git a/yarn-project/archiver/src/archiver/archiver_http_server.ts b/yarn-project/archiver/src/rpc/archiver_server.ts similarity index 95% rename from yarn-project/archiver/src/archiver/archiver_http_server.ts rename to yarn-project/archiver/src/rpc/archiver_server.ts index 09d39538a87..77cb761ae0b 100644 --- a/yarn-project/archiver/src/archiver/archiver_http_server.ts +++ b/yarn-project/archiver/src/rpc/archiver_server.ts @@ -10,7 +10,7 @@ import { import { EthAddress, Fr } from '@aztec/circuits.js'; import { JsonRpcServer } from '@aztec/foundation/json-rpc/server'; -import { Archiver } from './archiver.js'; +import { Archiver } from '../index.js'; /** * Wrap an Archiver instance with a JSON RPC HTTP server. diff --git a/yarn-project/archiver/src/rpc/index.ts b/yarn-project/archiver/src/rpc/index.ts new file mode 100644 index 00000000000..726d9120af8 --- /dev/null +++ b/yarn-project/archiver/src/rpc/index.ts @@ -0,0 +1,2 @@ +export * from './archiver_client.js'; +export * from './archiver_server.js'; diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 0e86055fee1..216febe889f 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -1,5 +1,4 @@ -import { ArchiveSource, Archiver, ArchiverDataStore, LMDBArchiverStore } from '@aztec/archiver'; -import { createArchiverClient } from '@aztec/aztec.js'; +import { ArchiveSource, Archiver, LMDBArchiverStore, createArchiverClient } from '@aztec/archiver'; import { AztecNode, ContractData, diff --git a/yarn-project/aztec.js/src/index.ts b/yarn-project/aztec.js/src/index.ts index 516f6c9245c..6d91e70c4de 100644 --- a/yarn-project/aztec.js/src/index.ts +++ b/yarn-project/aztec.js/src/index.ts @@ -48,7 +48,7 @@ export { waitForAccountSynch, } from './utils/index.js'; -export { createPXEClient, createAztecNodeClient, createArchiverClient } from './rpc_clients/index.js'; +export { createPXEClient } from './rpc_clients/index.js'; export { AuthWitnessProvider } from './account/index.js'; @@ -103,6 +103,7 @@ export { TxReceipt, TxStatus, UnencryptedL2Log, + createAztecNodeClient, emptyFunctionCall, merkleTreeIds, mockTx, diff --git a/yarn-project/aztec.js/src/rpc_clients/aztec_node_client.ts b/yarn-project/aztec.js/src/rpc_clients/aztec_node_client.ts deleted file mode 100644 index 7520eaee226..00000000000 --- a/yarn-project/aztec.js/src/rpc_clients/aztec_node_client.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { - AztecNode, - ContractData, - ExtendedContractData, - ExtendedUnencryptedL2Log, - L1ToL2MessageAndIndex, - L2Block, - L2BlockL2Logs, - L2Tx, - LogId, - Tx, - TxHash, -} from '@aztec/circuit-types'; -import { BlockHeader, FunctionSelector } from '@aztec/circuits.js'; -import { AztecAddress } from '@aztec/foundation/aztec-address'; -import { EthAddress } from '@aztec/foundation/eth-address'; -import { Fr } from '@aztec/foundation/fields'; -import { createJsonRpcClient, defaultFetch } from '@aztec/foundation/json-rpc/client'; -import { SiblingPath } from '@aztec/types/membership'; - -/** - * Creates a JSON-RPC client to remotely talk to an Aztec Node. - * @param url - The URL of the Aztec Node. - * @param fetch - The fetch implementation to use. - * @returns A JSON-RPC client of Aztec Node. - */ -export function createAztecNodeClient(url: string, fetch = defaultFetch): AztecNode { - return createJsonRpcClient( - url, - { - AztecAddress, - EthAddress, - ExtendedContractData, - ExtendedUnencryptedL2Log, - ContractData, - Fr, - FunctionSelector, - BlockHeader, - L2Block, - L2Tx, - LogId, - TxHash, - SiblingPath, - L1ToL2MessageAndIndex, - }, - { Tx, L2BlockL2Logs }, - false, - fetch, - ); -} diff --git a/yarn-project/aztec.js/src/rpc_clients/index.ts b/yarn-project/aztec.js/src/rpc_clients/index.ts index 43b4ba5cd33..8dfa33a26d6 100644 --- a/yarn-project/aztec.js/src/rpc_clients/index.ts +++ b/yarn-project/aztec.js/src/rpc_clients/index.ts @@ -1,3 +1 @@ export * from './pxe_client.js'; -export * from './aztec_node_client.js'; -export * from './archiver_client.js'; diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index c7e55554cf2..732bb4c5fd8 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -44,7 +44,10 @@ "commander": "^11.1.0", "koa": "^2.14.2", "koa-router": "^12.0.0", + "leveldown": "^6.1.1", + "lmdb": "^2.9.2", "lodash.pick": "^4.4.0", + "memdown": "^6.1.1", "viem": "^1.2.5", "winston": "^3.10.0", "winston-daily-rotate-file": "^4.7.1" diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index fef43b7f84a..1c30fe7990a 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -1,7 +1,8 @@ import { deployInitialTestAccounts } from '@aztec/accounts/testing'; import { Archiver, LMDBArchiverStore, createArchiverRpcServer, getConfigEnvVars } from '@aztec/archiver'; import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; -import { AccountManager, GrumpkinScalar, createAztecNodeClient, fileURLToPath } from '@aztec/aztec.js'; +import { AccountManager, GrumpkinScalar, fileURLToPath } from '@aztec/aztec.js'; +import { createAztecNodeClient } from '@aztec/circuit-types'; import { NULL_KEY } from '@aztec/ethereum'; import { startHttpRpcServer } from '@aztec/foundation/json-rpc/server'; import { DebugLogger, LogFn } from '@aztec/foundation/log'; diff --git a/yarn-project/aztec/src/cli/util.ts b/yarn-project/aztec/src/cli/util.ts index 49b7b646d59..33f68f78974 100644 --- a/yarn-project/aztec/src/cli/util.ts +++ b/yarn-project/aztec/src/cli/util.ts @@ -2,7 +2,6 @@ import { ArchiverConfig } from '@aztec/archiver'; import { LogFn } from '@aztec/foundation/log'; import { LevelDown, default as leveldown } from 'leveldown'; -import { LevelUp, default as levelup } from 'levelup'; import { RootDatabase, open } from 'lmdb'; import { MemDown, default as memdown } from 'memdown'; import { mkdir } from 'node:fs/promises'; @@ -11,14 +10,13 @@ import { join } from 'node:path'; export const createMemDown = () => (memdown as any)() as MemDown; export const createLevelDown = (path: string) => (leveldown as any)(path) as LevelDown; -const DB_SUBDIR = 'aztec-node-db'; -const WORLD_STATE_SUBDIR = 'aztec-world-state-db'; -const NODE_METADATA_KEY = '@@aztec_node_metadata'; +const DB_SUBDIR = 'aztec-archiver-db'; +const ARCHIVER_METADATA_KEY = '@@aztec_archiver_metadata'; /** * The metadata for an aztec node. */ -type NodeMetadata = { +type ArchiverMetadata = { /** * The address of the rollup contract on L1 */ @@ -26,52 +24,54 @@ type NodeMetadata = { }; /** - * Opens the database for the aztec node. If a data directory is specified, then this attempts to create it. - * @param config - The configuration to be used by the aztec node. + * Opens the database for the archiver. If a data directory is specified, then this attempts to create it. + * @param config - The configuration to be used by the archiver. * @throws If `config.dataDirectory` is set and the directory cannot be created. - * @returns The database for the aztec node. + * @returns The database instance for the archiver. */ export async function openDb(config: ArchiverConfig, log: LogFn): Promise { - const nodeMetadata: NodeMetadata = { + const archiverMetadata: ArchiverMetadata = { rollupContractAddress: config.l1Contracts.rollupAddress.toString(), }; - let nodeDb: RootDatabase; - let worldStateDb: LevelUp; + let archiverDb: RootDatabase; if (config.dataDirectory) { - const nodeDir = join(config.dataDirectory, DB_SUBDIR); + const dir = join(config.dataDirectory, DB_SUBDIR); // this throws if we don't have permissions to create the directory - await mkdir(nodeDir, { recursive: true }); + await mkdir(dir, { recursive: true }); - log(`Opening aztec-node database at ${nodeDir}`); - nodeDb = open(nodeDir, {}); + log(`Opening archiver database at ${dir}`); + archiverDb = open(dir, {}); } else { log('Opening temporary databases'); // not passing a path will use a temp file that gets deleted when the process exits - nodeDb = open({}); - worldStateDb = levelup(createMemDown()); + archiverDb = open({}); } - await checkNodeMetadataAndClear(nodeDb, nodeMetadata, log); - return nodeDb; + await checkArchiverMetadataAndClear(archiverDb, archiverMetadata, log); + return archiverDb; } /** - * Checks the node metadata and clears the database if the rollup contract address has changed. - * @param nodeDb - The database for the aztec node. - * @param nodeMetadata - The metadata for the aztec node. + * Checks the archiver metadata and clears the database if the rollup contract address has changed. + * @param archiverDb - The database for the aztec archiver. + * @param archiverMetadata - The metadata for the aztec archiver. */ -async function checkNodeMetadataAndClear(nodeDb: RootDatabase, nodeMetadata: NodeMetadata, log: LogFn): Promise { - const metadataDB = nodeDb.openDB('metadata', {}); +async function checkArchiverMetadataAndClear( + archiverDb: RootDatabase, + archiverMetadata: ArchiverMetadata, + log: LogFn, +): Promise { + const metadataDB = archiverDb.openDB('metadata', {}); try { - const existing = metadataDB.get(NODE_METADATA_KEY); + const existing = metadataDB.get(ARCHIVER_METADATA_KEY); // if the rollup addresses are different, wipe the local database and start over - if (!existing || existing.rollupContractAddress !== nodeMetadata.rollupContractAddress) { + if (!existing || existing.rollupContractAddress !== archiverMetadata.rollupContractAddress) { log('Rollup contract address has changed, clearing databases'); - await nodeDb.clearAsync(); + await archiverDb.clearAsync(); } - await metadataDB.put(NODE_METADATA_KEY, nodeMetadata); + await metadataDB.put(ARCHIVER_METADATA_KEY, archiverMetadata); } finally { await metadataDB.close(); } diff --git a/yarn-project/end-to-end/package.json b/yarn-project/end-to-end/package.json index 891f116ed8c..bd1e835e57d 100644 --- a/yarn-project/end-to-end/package.json +++ b/yarn-project/end-to-end/package.json @@ -25,7 +25,6 @@ "dependencies": { "@aztec/accounts": "workspace:^", "@aztec/archiver": "workspace:^", - "@aztec/aztec": "workspace:^", "@aztec/aztec-node": "workspace:^", "@aztec/aztec.js": "workspace:^", "@aztec/circuit-types": "workspace:^", diff --git a/yarn-project/end-to-end/tsconfig.json b/yarn-project/end-to-end/tsconfig.json index 8e251af3bd1..2cd953c345d 100644 --- a/yarn-project/end-to-end/tsconfig.json +++ b/yarn-project/end-to-end/tsconfig.json @@ -12,9 +12,6 @@ { "path": "../archiver" }, - { - "path": "../aztec" - }, { "path": "../aztec-node" }, diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 79eac44ef2a..1ad2d52786c 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -253,7 +253,7 @@ __metadata: languageName: unknown linkType: soft -"@aztec/aztec@workspace:^, @aztec/aztec@workspace:aztec": +"@aztec/aztec@workspace:aztec": version: 0.0.0-use.local resolution: "@aztec/aztec@workspace:aztec" dependencies: @@ -278,7 +278,10 @@ __metadata: jest: ^29.5.0 koa: ^2.14.2 koa-router: ^12.0.0 + leveldown: ^6.1.1 + lmdb: ^2.9.2 lodash.pick: ^4.4.0 + memdown: ^6.1.1 ts-jest: ^29.1.0 ts-node: ^10.9.1 typescript: ^5.0.4 @@ -406,7 +409,6 @@ __metadata: dependencies: "@aztec/accounts": "workspace:^" "@aztec/archiver": "workspace:^" - "@aztec/aztec": "workspace:^" "@aztec/aztec-node": "workspace:^" "@aztec/aztec.js": "workspace:^" "@aztec/circuit-types": "workspace:^" @@ -9475,7 +9477,7 @@ __metadata: languageName: node linkType: hard -"lmdb@npm:^2.9.1": +"lmdb@npm:^2.9.1, lmdb@npm:^2.9.2": version: 2.9.2 resolution: "lmdb@npm:2.9.2" dependencies: From 11811675005d1f0c8f0a16cb4834f1135e292ec5 Mon Sep 17 00:00:00 2001 From: spypsy Date: Mon, 15 Jan 2024 17:29:26 +0000 Subject: [PATCH 19/50] multi-class rpc server --- .../archiver/src/rpc/archiver_client.ts | 14 ++---- .../aztec.js/src/rpc_clients/pxe_client.ts | 1 + yarn-project/aztec/src/aztec_client.ts | 0 .../src/aztec_node/rpc/aztec_node_client.ts | 1 + .../json-rpc/client/json_rpc_client.test.ts | 2 +- .../src/json-rpc/client/json_rpc_client.ts | 12 ++++- .../src/json-rpc/server/json_proxy.ts | 27 +++++++++- .../src/json-rpc/server/json_rpc_server.ts | 49 ++++++++++++++++++- 8 files changed, 91 insertions(+), 15 deletions(-) create mode 100644 yarn-project/aztec/src/aztec_client.ts diff --git a/yarn-project/archiver/src/rpc/archiver_client.ts b/yarn-project/archiver/src/rpc/archiver_client.ts index a3d041a9e72..82fb795f214 100644 --- a/yarn-project/archiver/src/rpc/archiver_client.ts +++ b/yarn-project/archiver/src/rpc/archiver_client.ts @@ -1,24 +1,19 @@ import { ContractData, - ContractDataSource, EncodedContractFunction, ExtendedContractData, ExtendedUnencryptedL2Log, L1ToL2Message, - L1ToL2MessageSource, L2Block, L2BlockL2Logs, - L2BlockSource, - L2LogsSource, } from '@aztec/circuit-types'; import { EthAddress, Fr } from '@aztec/circuits.js'; import { createJsonRpcClient, makeFetch } from '@aztec/foundation/json-rpc/client'; -export const createArchiverClient = ( - url: string, - fetch = makeFetch([1, 2, 3], true), -): L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource => - createJsonRpcClient( +import { ArchiveSource } from '../index.js'; + +export const createArchiverClient = (url: string, fetch = makeFetch([1, 2, 3], true)): ArchiveSource => + createJsonRpcClient( url, { ContractData, @@ -33,5 +28,6 @@ export const createArchiverClient = ( }, {}, false, + false, fetch, ); diff --git a/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts b/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts index 38045e6ad0a..7c49f037084 100644 --- a/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts +++ b/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts @@ -57,5 +57,6 @@ export const createPXEClient = (url: string, fetch = makeFetch([1, 2, 3], true)) }, { Tx, TxReceipt, L2BlockL2Logs }, false, + false, fetch, ); diff --git a/yarn-project/aztec/src/aztec_client.ts b/yarn-project/aztec/src/aztec_client.ts new file mode 100644 index 00000000000..e69de29bb2d diff --git a/yarn-project/circuit-types/src/aztec_node/rpc/aztec_node_client.ts b/yarn-project/circuit-types/src/aztec_node/rpc/aztec_node_client.ts index 3a2f9ebb635..0ea76ae221d 100644 --- a/yarn-project/circuit-types/src/aztec_node/rpc/aztec_node_client.ts +++ b/yarn-project/circuit-types/src/aztec_node/rpc/aztec_node_client.ts @@ -42,6 +42,7 @@ export function createAztecNodeClient(url: string, fetch = defaultFetch): AztecN }, { Tx, L2BlockL2Logs }, false, + false, fetch, ); } diff --git a/yarn-project/foundation/src/json-rpc/client/json_rpc_client.test.ts b/yarn-project/foundation/src/json-rpc/client/json_rpc_client.test.ts index 7b720a6e123..2e2c61aa02b 100644 --- a/yarn-project/foundation/src/json-rpc/client/json_rpc_client.test.ts +++ b/yarn-project/foundation/src/json-rpc/client/json_rpc_client.test.ts @@ -10,7 +10,7 @@ it('test an RPC function over client', async () => { const result = await request(server.getApp().callback()).post(`/${method}`).send(body); return JSON.parse(result.text); }; - const client = createJsonRpcClient('', { TestNote }, {}, true, mockFetch); + const client = createJsonRpcClient('', { TestNote }, {}, true, false, mockFetch); const result = await client.addNotes([new TestNote('c')]); expect(result[0]).toBeInstanceOf(TestNote); expect(result[1]).toBeInstanceOf(TestNote); diff --git a/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts b/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts index a7cb59263ee..ea4f2258061 100644 --- a/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts +++ b/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts @@ -88,12 +88,18 @@ export function makeFetch(retries: number[], noRetry: boolean, log?: DebugLogger /** * Creates a Proxy object that delegates over RPC and satisfies RemoteObject. * The server should have ran new JsonRpcServer(). + * @param host - The host URL. + * @param stringClassMap - A map of class names to string representations. + * @param objectClassMap - A map of class names to class constructors. + * @param useApiEndpoints - Whether to use the API endpoints or the default RPC endpoint. + * @param fetch - The fetch implementation to use. */ export function createJsonRpcClient( host: string, stringClassMap: StringClassConverterInput, objectClassMap: JsonClassConverterInput, useApiEndpoints: boolean, + namespaceMethods?: string | false, fetch = defaultFetch, ) { const classConverter = new ClassConverter(stringClassMap, objectClassMap); @@ -122,7 +128,11 @@ export function createJsonRpcClient( return new Proxy( {}, { - get: (target, rpcMethod: string) => { + get: (target, method: string) => { + let rpcMethod = method; + if (namespaceMethods) { + rpcMethod = `${namespaceMethods}_${method}`; + } if (['then', 'catch'].includes(rpcMethod)) { return Reflect.get(target, rpcMethod); } diff --git a/yarn-project/foundation/src/json-rpc/server/json_proxy.ts b/yarn-project/foundation/src/json-rpc/server/json_proxy.ts index e7630d7f5e8..8ca9ba70cb2 100644 --- a/yarn-project/foundation/src/json-rpc/server/json_proxy.ts +++ b/yarn-project/foundation/src/json-rpc/server/json_proxy.ts @@ -7,6 +7,13 @@ import { assert, hasOwnProperty } from '../js_utils.js'; const debug = createDebugLogger('json-rpc:json_proxy'); +export type ClassMaps = { + /** The String class map */ + stringClassMap: StringClassConverterInput; + /** The object class map */ + objectClassMap: JsonClassConverterInput; +}; + /** * Handles conversion of objects over the write. * Delegates to a ClassConverter object. @@ -15,8 +22,8 @@ export class JsonProxy { classConverter: ClassConverter; constructor( private handler: object, - stringClassMap: StringClassConverterInput, - objectClassMap: JsonClassConverterInput, + private stringClassMap: StringClassConverterInput, + private objectClassMap: JsonClassConverterInput, ) { this.classConverter = new ClassConverter(stringClassMap, objectClassMap); } @@ -40,4 +47,20 @@ export class JsonProxy { debug(format('JsonProxy:call', methodName, '->', ret)); return ret; } + + /** + * Get a list of methods. + * @returns A list of methods. + */ + public getMethods(): string[] { + return Object.getOwnPropertyNames(Object.getPrototypeOf(this.handler)); + } + + /** + * Gets the class maps that were used to create the proxy. + * @returns The string & object class maps. + */ + public getClassMaps(): ClassMaps { + return { stringClassMap: this.stringClassMap, objectClassMap: this.objectClassMap }; + } } diff --git a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts index ea1bd7e2d05..2db87b6c833 100644 --- a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts +++ b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts @@ -8,7 +8,7 @@ import Router from 'koa-router'; import { createDebugLogger } from '../../log/index.js'; import { JsonClassConverterInput, StringClassConverterInput } from '../class_converter.js'; import { convertBigintsInObj } from '../convert.js'; -import { JsonProxy } from './json_proxy.js'; +import { ClassMaps, JsonProxy } from './json_proxy.js'; /** * JsonRpcServer. @@ -21,7 +21,8 @@ export class JsonRpcServer { stringClassMap: StringClassConverterInput, objectClassMap: JsonClassConverterInput, private createApi: boolean, - private disallowedMethods: string[] = [], + /** List of methods to disallow from calling remotely */ + public readonly disallowedMethods: string[] = [], private log = createDebugLogger('aztec:foundation:json-rpc:server'), ) { this.proxy = new JsonProxy(handler, stringClassMap, objectClassMap); @@ -223,3 +224,47 @@ export function startHttpRpcServer( return httpServer; } +/** + * List of namespace to server instance. + */ +type ServerList = { + /** name of the service to be used for namespacing */ + name: string; + /** server instance */ + server: JsonRpcServer; +}[]; + +/** + * Creates a single JsonRpcServer from multiple servers. + * @param servers - List of servers to be combined into a single server, passed as ServerList. + * @returns A single JsonRpcServer with namespaced methods. + */ +export function createMultiJsonRpcServer( + servers: ServerList, + log = createDebugLogger('aztec:foundation:json-rpc:multi-server'), +): JsonRpcServer { + const handler = {} as any; + const disallowedMethods: string[] = []; + + for (const { name: namespace, server } of servers) { + const serverMethods = server.proxy.getMethods(); + + for (const method of serverMethods) { + const namespacedMethod = `${namespace}_${method}`; + + handler[namespacedMethod] = (...args: any[]) => { + return server.proxy.call(method, ...args); + }; + } + + // get the combined disallowed methods from all servers. + disallowedMethods.push(...server.disallowedMethods.map(method => `${namespace}_${method}`)); + } + + // Get the combined stringClassMap & objectClassMap from all servers + const classMaps = servers.reduce((acc, { server }) => { + return { ...acc, ...server.proxy.getClassMaps() }; + }, {} as ClassMaps); + + return new JsonRpcServer(handler, classMaps.stringClassMap, classMaps.objectClassMap, false, [], log); +} From 524d9d26c80fd827948e87001f3dc4c4751bd233 Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 16 Jan 2024 10:11:28 +0000 Subject: [PATCH 20/50] WIP using multi json rpc server --- yarn-project/aztec/src/cli/cli.ts | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index 1c30fe7990a..cfa0a266885 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -4,13 +4,14 @@ import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeC import { AccountManager, GrumpkinScalar, fileURLToPath } from '@aztec/aztec.js'; import { createAztecNodeClient } from '@aztec/circuit-types'; import { NULL_KEY } from '@aztec/ethereum'; -import { startHttpRpcServer } from '@aztec/foundation/json-rpc/server'; +import { createMultiJsonRpcServer, startHttpRpcServer } from '@aztec/foundation/json-rpc/server'; import { DebugLogger, LogFn } from '@aztec/foundation/log'; import { BootstrapNode, P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; import { PXEService, PXEServiceConfig, createPXERpcServer, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import { Command } from 'commander'; import { readFileSync } from 'fs'; +import http from 'http'; import pick from 'lodash.pick'; import { dirname, resolve } from 'path'; import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; @@ -21,9 +22,10 @@ import { cliTexts } from './texts.js'; import { openDb } from './util.js'; const { + AZTEC_PORT = '80', AZTEC_NODE_PORT = '8079', PXE_PORT = '8080', - ARCHIVER_PORT = '80', + ARCHIVER_PORT = '8080', AZTEC_NODE_URL, DEPLOY_AZTEC_CONTRACTS, } = process.env; @@ -94,8 +96,9 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { program .command('sandbox') .description('Starts Aztec sandbox.') - .option('-np, --node-port ', 'Port to run Aztec Node on.', AZTEC_NODE_PORT) - .option('-pp, --pxe-port ', 'Port to run PXE on (optional).', PXE_PORT) + .option('-p, --port ', 'Port to run Aztec on.', AZTEC_PORT) + // .option('-np, --node-port ', 'Port to run Aztec Node on.', AZTEC_NODE_PORT) + // .option('-pp, --pxe-port ', 'Port to run PXE on (optional).', PXE_PORT) .option('-s, --skip-test-accounts', 'DO NOT deploy test accounts.', false) // .option() .action(async options => { @@ -116,11 +119,20 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { } } - // Start Node and PXE JSON-RPC servers - startHttpRpcServer(node, createAztecNodeRpcServer, options.nodePort); - userLog(`Aztec Node JSON-RPC Server listening on nodePort ${options.nodePort}`); - startHttpRpcServer(pxe, createPXERpcServer, options.pxePort); - userLog(`PXE JSON-RPC Server listening on port ${options.pxePort}`); + // Start Node and PXE JSON-RPC server + const nodeServer = createAztecNodeRpcServer(node); + const pxeServer = createPXERpcServer(pxe); + const rpcServer = createMultiJsonRpcServer([{ node: nodeServer }, { pxe: pxeServer }], debugLogger); + + const app = rpcServer.getApp(); + const httpServer = http.createServer(app.callback()); + httpServer.listen(options.port); + userLog(`Aztec Server listening on port ${options.pxePort}`); + + // startHttpRpcServer(node, createAztecNodeRpcServer, options.nodePort); + // userLog(`Aztec Node JSON-RPC Server listening on nodePort ${options.nodePort}`); + // startHttpRpcServer(pxe, createPXERpcServer, options.pxePort); + // userLog(`PXE JSON-RPC Server listening on port ${options.pxePort}`); }); // Start Aztec modules with options From 67d9e06b5de90483b4de611baade225646992bb9 Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 16 Jan 2024 12:24:29 +0000 Subject: [PATCH 21/50] JSON RPC fixes --- yarn-project/aztec/src/cli/cli.ts | 2 +- .../foundation/src/json-rpc/server/index.ts | 2 +- .../src/json-rpc/server/json_proxy.ts | 16 ----- .../src/json-rpc/server/json_rpc_server.ts | 66 +++++++++++++++---- .../pxe/src/synchronizer/synchronizer.ts | 2 + 5 files changed, 58 insertions(+), 30 deletions(-) diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index cfa0a266885..33e8b16fe9c 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -127,7 +127,7 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { const app = rpcServer.getApp(); const httpServer = http.createServer(app.callback()); httpServer.listen(options.port); - userLog(`Aztec Server listening on port ${options.pxePort}`); + userLog(`Aztec Server listening on port ${options.port}`); // startHttpRpcServer(node, createAztecNodeRpcServer, options.nodePort); // userLog(`Aztec Node JSON-RPC Server listening on nodePort ${options.nodePort}`); diff --git a/yarn-project/foundation/src/json-rpc/server/index.ts b/yarn-project/foundation/src/json-rpc/server/index.ts index a20b679993b..7937ba0e703 100644 --- a/yarn-project/foundation/src/json-rpc/server/index.ts +++ b/yarn-project/foundation/src/json-rpc/server/index.ts @@ -1,2 +1,2 @@ -export { JsonRpcServer, createStatusRouter, startHttpRpcServer } from './json_rpc_server.js'; +export { JsonRpcServer, createStatusRouter, startHttpRpcServer, createMultiJsonRpcServer } from './json_rpc_server.js'; export { JsonProxy } from './json_proxy.js'; diff --git a/yarn-project/foundation/src/json-rpc/server/json_proxy.ts b/yarn-project/foundation/src/json-rpc/server/json_proxy.ts index 8ca9ba70cb2..bad680e04d4 100644 --- a/yarn-project/foundation/src/json-rpc/server/json_proxy.ts +++ b/yarn-project/foundation/src/json-rpc/server/json_proxy.ts @@ -47,20 +47,4 @@ export class JsonProxy { debug(format('JsonProxy:call', methodName, '->', ret)); return ret; } - - /** - * Get a list of methods. - * @returns A list of methods. - */ - public getMethods(): string[] { - return Object.getOwnPropertyNames(Object.getPrototypeOf(this.handler)); - } - - /** - * Gets the class maps that were used to create the proxy. - * @returns The string & object class maps. - */ - public getClassMaps(): ClassMaps { - return { stringClassMap: this.stringClassMap, objectClassMap: this.objectClassMap }; - } } diff --git a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts index 2db87b6c833..35327cfb2f8 100644 --- a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts +++ b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts @@ -15,11 +15,14 @@ import { ClassMaps, JsonProxy } from './json_proxy.js'; * Minimal, dev-friendly mechanism to create a server from an object. */ export class JsonRpcServer { - proxy: JsonProxy; + /** + * The proxy object. + */ + public proxy: JsonProxy; constructor( private handler: object, - stringClassMap: StringClassConverterInput, - objectClassMap: JsonClassConverterInput, + private stringClassMap: StringClassConverterInput, + private objectClassMap: JsonClassConverterInput, private createApi: boolean, /** List of methods to disallow from calling remotely */ public readonly disallowedMethods: string[] = [], @@ -188,6 +191,32 @@ export class JsonRpcServer { const httpServer = http.createServer(this.getApp(prefix).callback()); httpServer.listen(port); } + + /** + * Get a list of methods. + * @returns A list of methods. + */ + public getMethods(): string[] { + return Object.getOwnPropertyNames(Object.getPrototypeOf(this.handler)); + } + + /** + * Gets the class maps that were used to create the proxy. + * @returns The string & object class maps. + */ + public getClassMaps(): ClassMaps { + return { stringClassMap: this.stringClassMap, objectClassMap: this.objectClassMap }; + } + + /** + * Call an RPC method. + * @param methodName - The RPC method. + * @param jsonParams - The RPG parameters. + * @returns The remote result. + */ + public async call(methodName: string, jsonParams: any[] = []) { + return await this.proxy.call(methodName, jsonParams); + } } /** @@ -229,9 +258,7 @@ export function startHttpRpcServer( */ type ServerList = { /** name of the service to be used for namespacing */ - name: string; - /** server instance */ - server: JsonRpcServer; + [name: string]: JsonRpcServer; }[]; /** @@ -245,26 +272,41 @@ export function createMultiJsonRpcServer( ): JsonRpcServer { const handler = {} as any; const disallowedMethods: string[] = []; + const classMapsArr: ClassMaps[] = []; - for (const { name: namespace, server } of servers) { - const serverMethods = server.proxy.getMethods(); + for (const serverEntry of servers) { + const [namespace, server] = Object.entries(serverEntry)[0]; + console.log('namespace', namespace, 'server', server); + const serverMethods = server.getMethods(); for (const method of serverMethods) { const namespacedMethod = `${namespace}_${method}`; handler[namespacedMethod] = (...args: any[]) => { - return server.proxy.call(method, ...args); + return server.call(method, ...args); }; } // get the combined disallowed methods from all servers. disallowedMethods.push(...server.disallowedMethods.map(method => `${namespace}_${method}`)); + // get the combined classmaps from all servers. + const classMap = server.getClassMaps(); + classMapsArr.push({ + stringClassMap: classMap.stringClassMap, + objectClassMap: classMap.objectClassMap, + }); } // Get the combined stringClassMap & objectClassMap from all servers - const classMaps = servers.reduce((acc, { server }) => { - return { ...acc, ...server.proxy.getClassMaps() }; - }, {} as ClassMaps); + const classMaps = classMapsArr.reduce( + (acc, curr) => { + return { + stringClassMap: { ...acc.stringClassMap, ...curr.stringClassMap }, + objectClassMap: { ...acc.objectClassMap, ...curr.objectClassMap }, + }; + }, + { stringClassMap: {}, objectClassMap: {} } as ClassMaps, + ); return new JsonRpcServer(handler, classMaps.stringClassMap, classMaps.objectClassMap, false, [], log); } diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.ts b/yarn-project/pxe/src/synchronizer/synchronizer.ts index d0088fd9cc4..e2cfab1ae7c 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.ts @@ -148,7 +148,9 @@ export class Synchronizer { const logCount = L2BlockL2Logs.getTotalLogCount(encryptedLogs); this.log(`Forwarding ${logCount} encrypted logs and blocks to ${this.noteProcessors.length} note processors`); + let i = 1; for (const noteProcessor of this.noteProcessors) { + console.log('processing note processor', i++); await noteProcessor.process(blockContexts, encryptedLogs); } return true; From a11863868cb539bf367be42c4ba1c6f58e1dca6f Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 16 Jan 2024 14:53:20 +0000 Subject: [PATCH 22/50] debug logging --- yarn-project/aztec.js/src/rpc_clients/pxe_client.ts | 2 +- yarn-project/aztec/src/cli/cli.ts | 2 +- yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts | 1 - yarn-project/pxe/src/synchronizer/synchronizer.ts | 1 + 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts b/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts index 7c49f037084..296daead28b 100644 --- a/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts +++ b/yarn-project/aztec.js/src/rpc_clients/pxe_client.ts @@ -57,6 +57,6 @@ export const createPXEClient = (url: string, fetch = makeFetch([1, 2, 3], true)) }, { Tx, TxReceipt, L2BlockL2Logs }, false, - false, + 'pxe', fetch, ); diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index 33e8b16fe9c..d320eabe835 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -22,7 +22,7 @@ import { cliTexts } from './texts.js'; import { openDb } from './util.js'; const { - AZTEC_PORT = '80', + AZTEC_PORT = '8080', AZTEC_NODE_PORT = '8079', PXE_PORT = '8080', ARCHIVER_PORT = '8080', diff --git a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts index 35327cfb2f8..81a44e01002 100644 --- a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts +++ b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts @@ -276,7 +276,6 @@ export function createMultiJsonRpcServer( for (const serverEntry of servers) { const [namespace, server] = Object.entries(serverEntry)[0]; - console.log('namespace', namespace, 'server', server); const serverMethods = server.getMethods(); for (const method of serverMethods) { diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.ts b/yarn-project/pxe/src/synchronizer/synchronizer.ts index e2cfab1ae7c..e3fdfe2b264 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.ts @@ -155,6 +155,7 @@ export class Synchronizer { } return true; } catch (err) { + console.log(err); this.log.error(`Error in synchronizer work`, err); return false; } From e1aa2d6b14ed9000506797fb55110a4aa07dd6a1 Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 16 Jan 2024 16:48:50 +0000 Subject: [PATCH 23/50] update lmdb --- yarn-project/archiver/package.json | 2 +- yarn-project/aztec-node/package.json | 2 +- yarn-project/end-to-end/package.json | 2 +- yarn-project/kv-store/package.json | 2 +- yarn-project/yarn.lock | 10 +++++----- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/yarn-project/archiver/package.json b/yarn-project/archiver/package.json index 4ebbd12dfdb..666993814c3 100644 --- a/yarn-project/archiver/package.json +++ b/yarn-project/archiver/package.json @@ -41,7 +41,7 @@ "@aztec/l1-artifacts": "workspace:^", "@types/lodash.omit": "^4.5.7", "debug": "^4.3.4", - "lmdb": "^2.9.1", + "lmdb": "^2.9.2", "lodash.omit": "^4.5.0", "tsc-watch": "^6.0.0", "tslib": "^2.5.0", diff --git a/yarn-project/aztec-node/package.json b/yarn-project/aztec-node/package.json index e06bf2d75ed..50fc850ceba 100644 --- a/yarn-project/aztec-node/package.json +++ b/yarn-project/aztec-node/package.json @@ -49,7 +49,7 @@ "koa": "^2.14.2", "koa-router": "^12.0.0", "levelup": "^5.1.1", - "lmdb": "^2.9.1", + "lmdb": "^2.9.2", "memdown": "^6.1.1", "tslib": "^2.4.0" }, diff --git a/yarn-project/end-to-end/package.json b/yarn-project/end-to-end/package.json index bd1e835e57d..f0d25e2a6db 100644 --- a/yarn-project/end-to-end/package.json +++ b/yarn-project/end-to-end/package.json @@ -58,7 +58,7 @@ "koa": "^2.14.2", "koa-static": "^5.0.0", "levelup": "^5.1.1", - "lmdb": "^2.9.1", + "lmdb": "^2.9.2", "lodash.compact": "^3.0.1", "lodash.every": "^4.6.0", "lodash.times": "^4.3.2", diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json index 8aea043378d..4c440901969 100644 --- a/yarn-project/kv-store/package.json +++ b/yarn-project/kv-store/package.json @@ -26,7 +26,7 @@ }, "dependencies": { "@aztec/foundation": "workspace:^", - "lmdb": "^2.9.1" + "lmdb": "^2.9.2" }, "devDependencies": { "@jest/globals": "^29.5.0", diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 1ad2d52786c..b62469de6d5 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -133,7 +133,7 @@ __metadata: debug: ^4.3.4 jest: ^29.5.0 jest-mock-extended: ^3.0.4 - lmdb: ^2.9.1 + lmdb: ^2.9.2 lodash.omit: ^4.5.0 lodash.times: ^4.3.2 ts-jest: ^29.1.0 @@ -195,7 +195,7 @@ __metadata: koa: ^2.14.2 koa-router: ^12.0.0 levelup: ^5.1.1 - lmdb: ^2.9.1 + lmdb: ^2.9.2 memdown: ^6.1.1 ts-jest: ^29.1.0 ts-node: ^10.9.1 @@ -443,7 +443,7 @@ __metadata: koa: ^2.14.2 koa-static: ^5.0.0 levelup: ^5.1.1 - lmdb: ^2.9.1 + lmdb: ^2.9.2 lodash.compact: ^3.0.1 lodash.every: ^4.6.0 lodash.times: ^4.3.2 @@ -573,7 +573,7 @@ __metadata: "@types/node": ^18.7.23 jest: ^29.5.0 jest-mock-extended: ^3.0.3 - lmdb: ^2.9.1 + lmdb: ^2.9.2 ts-jest: ^29.1.0 ts-node: ^10.9.1 typescript: ^5.0.4 @@ -9477,7 +9477,7 @@ __metadata: languageName: node linkType: hard -"lmdb@npm:^2.9.1, lmdb@npm:^2.9.2": +"lmdb@npm:^2.9.2": version: 2.9.2 resolution: "lmdb@npm:2.9.2" dependencies: From 859743b6fcd1ad3562771d31ce8e119ad9b290ff Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Tue, 16 Jan 2024 19:02:22 +0000 Subject: [PATCH 24/50] fix: pxe notes crashing spuriously (#4049) This PR refactors how the PXE stores notes. They're now stored by their index in the note hash tree which should provide both uniqueness and a sorted order for notes. This gets rid of the "nullifiedNotes" set so reading and removing notes should be cleaner and more efficient. --- .../pxe/src/database/kv_pxe_database.ts | 131 ++++++++++-------- 1 file changed, 71 insertions(+), 60 deletions(-) diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index a6216e9be06..a8f1bb47ac5 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -1,5 +1,6 @@ import { ContractDao, MerkleTreeId, NoteFilter, PublicKey } from '@aztec/circuit-types'; import { AztecAddress, BlockHeader, CompleteAddress } from '@aztec/circuits.js'; +import { toBufferBE } from '@aztec/foundation/bigint-buffer'; import { Fr, Point } from '@aztec/foundation/fields'; import { AztecArray, AztecKVStore, AztecMap, AztecMultiMap, AztecSingleton } from '@aztec/kv-store'; @@ -27,18 +28,21 @@ export class KVPxeDatabase implements PxeDatabase { #authWitnesses: AztecMap; #capsules: AztecArray; #contracts: AztecMap; - #notes: AztecArray; - #nullifiedNotes: AztecMap; - #notesByContract: AztecMultiMap; - #notesByStorageSlot: AztecMultiMap; - #notesByTxHash: AztecMultiMap; - #notesByOwner: AztecMultiMap; - #deferredNotes: AztecArray; + + #notes: AztecMap; + #notesByNullifier: AztecMap; + #notesByContract: AztecMultiMap; + #notesByStorageSlot: AztecMultiMap; + #notesByTxHash: AztecMultiMap; + #notesByOwner: AztecMultiMap; + + #deferredNotes: AztecArray; #deferredNotesByContract: AztecMultiMap; #syncedBlockPerPublicKey: AztecMap; + #db: AztecKVStore; - constructor(db: AztecKVStore) { + constructor(private db: AztecKVStore) { this.#db = db; this.#addresses = db.createArray('addresses'); @@ -51,9 +55,8 @@ export class KVPxeDatabase implements PxeDatabase { this.#synchronizedBlock = db.createSingleton('block_header'); this.#syncedBlockPerPublicKey = db.createMap('synced_block_per_public_key'); - this.#notes = db.createArray('notes'); - this.#nullifiedNotes = db.createMap('nullified_notes'); - + this.#notes = db.createMap('notes'); + this.#notesByNullifier = db.createMap('notes'); this.#notesByContract = db.createMultiMap('notes_by_contract'); this.#notesByStorageSlot = db.createMultiMap('notes_by_storage_slot'); this.#notesByTxHash = db.createMultiMap('notes_by_tx_hash'); @@ -88,17 +91,22 @@ export class KVPxeDatabase implements PxeDatabase { await this.addNotes([note]); } - async addNotes(notes: NoteDao[]): Promise { - const newLength = await this.#notes.push(...notes.map(note => note.toBuffer())); - for (const [index, note] of notes.entries()) { - const noteId = newLength - notes.length + index; - await Promise.all([ - this.#notesByContract.set(note.contractAddress.toString(), noteId), - this.#notesByStorageSlot.set(note.storageSlot.toString(), noteId), - this.#notesByTxHash.set(note.txHash.toString(), noteId), - this.#notesByOwner.set(note.publicKey.toString(), noteId), - ]); - } + addNotes(notes: NoteDao[]): Promise { + return this.db.transaction(() => { + for (const dao of notes) { + // store notes by their index in the notes hash tree + // this provides the uniqueness we need to store individual notes + // and should also return notes in the order that they were created. + // Had we stored them by their nullifier, they would be returned in random order + const noteIndex = toBufferBE(dao.index, 32).toString(); + void this.#notes.set(noteIndex, dao.toBuffer()); + void this.#notesByNullifier.set(dao.siloedNullifier.toString(), noteIndex); + void this.#notesByContract.set(dao.contractAddress.toString(), noteIndex); + void this.#notesByStorageSlot.set(dao.storageSlot.toString(), noteIndex); + void this.#notesByTxHash.set(dao.txHash.toString(), noteIndex); + void this.#notesByOwner.set(dao.publicKey.toString(), noteIndex); + } + }); } async addDeferredNotes(deferredNotes: DeferredNoteDao[]): Promise { @@ -129,11 +137,6 @@ export class KVPxeDatabase implements PxeDatabase { * Removes all deferred notes for a given contract address. * @param contractAddress - the contract address to remove deferred notes for * @returns an array of the removed deferred notes - * - * @remarks We only remove indices from the deferred notes by contract map, but not the actual deferred notes. - * This is safe because our only getter for deferred notes is by contract address. - * If we should add a more general getter, we will need a delete vector for deferred notes as well, - * analogous to this.#nullifiedNotes. */ removeDeferredNotesByContract(contractAddress: AztecAddress): Promise { return this.#db.transaction(() => { @@ -149,25 +152,16 @@ export class KVPxeDatabase implements PxeDatabase { } void this.#deferredNotesByContract.deleteValue(contractAddress.toString(), index); + void this.#deferredNotes.setAt(index, null); } return deferredNotes; }); } - *#getAllNonNullifiedNotes(): IterableIterator { - for (const [index, serialized] of this.#notes.entries()) { - if (this.#nullifiedNotes.has(index)) { - continue; - } - - yield NoteDao.fromBuffer(serialized); - } - } - - async getNotes(filter: NoteFilter): Promise { + #getNotes(filter: NoteFilter = {}): NoteDao[] { const publicKey: PublicKey | undefined = filter.owner - ? (await this.getCompleteAddress(filter.owner))?.publicKey + ? this.#getCompleteAddress(filter.owner)?.publicKey : undefined; const initialNoteIds = publicKey @@ -178,15 +172,11 @@ export class KVPxeDatabase implements PxeDatabase { ? this.#notesByContract.getValues(filter.contractAddress.toString()) : filter.storageSlot ? this.#notesByStorageSlot.getValues(filter.storageSlot.toString()) - : undefined; - - if (!initialNoteIds) { - return Array.from(this.#getAllNonNullifiedNotes()); - } + : this.#notes.keys(); const result: NoteDao[] = []; for (const noteId of initialNoteIds) { - const serializedNote = this.#notes.at(noteId); + const serializedNote = this.#notes.get(noteId); if (!serializedNote) { continue; } @@ -214,26 +204,43 @@ export class KVPxeDatabase implements PxeDatabase { return result; } + getNotes(filter: NoteFilter): Promise { + return Promise.resolve(this.#getNotes(filter)); + } + removeNullifiedNotes(nullifiers: Fr[], account: PublicKey): Promise { if (nullifiers.length === 0) { return Promise.resolve([]); } - const nullifierSet = new Set(nullifiers.map(n => n.toString())); + return this.#db.transaction(() => { - const notesIds = this.#notesByOwner.getValues(account.toString()); const nullifiedNotes: NoteDao[] = []; - for (const noteId of notesIds) { - const note = NoteDao.fromBuffer(this.#notes.at(noteId)!); - if (nullifierSet.has(note.siloedNullifier.toString())) { - nullifiedNotes.push(note); + for (const nullifier of nullifiers) { + const noteIndex = this.#notesByNullifier.get(nullifier.toString()); + if (!noteIndex) { + continue; + } + + const noteBuffer = noteIndex ? this.#notes.get(noteIndex) : undefined; + + if (!noteBuffer) { + // note doesn't exist. Maybe it got nullified already + continue; + } - void this.#nullifiedNotes.set(noteId, true); - void this.#notesByOwner.deleteValue(account.toString(), noteId); - void this.#notesByTxHash.deleteValue(note.txHash.toString(), noteId); - void this.#notesByContract.deleteValue(note.contractAddress.toString(), noteId); - void this.#notesByStorageSlot.deleteValue(note.storageSlot.toString(), noteId); + const note = NoteDao.fromBuffer(noteBuffer); + if (!note.publicKey.equals(account)) { + // tried to nullify someone else's note + continue; } + + nullifiedNotes.push(note); + + void this.#notesByOwner.deleteValue(account.toString(), noteIndex); + void this.#notesByTxHash.deleteValue(note.txHash.toString(), noteIndex); + void this.#notesByContract.deleteValue(note.contractAddress.toString(), noteIndex); + void this.#notesByStorageSlot.deleteValue(note.storageSlot.toString(), noteIndex); } return nullifiedNotes; @@ -320,14 +327,18 @@ export class KVPxeDatabase implements PxeDatabase { }); } - getCompleteAddress(address: AztecAddress): Promise { + #getCompleteAddress(address: AztecAddress): CompleteAddress | undefined { const index = this.#addressIndex.get(address.toString()); if (typeof index === 'undefined') { - return Promise.resolve(undefined); + return undefined; } const value = this.#addresses.at(index); - return Promise.resolve(value ? CompleteAddress.fromBuffer(value) : undefined); + return value ? CompleteAddress.fromBuffer(value) : undefined; + } + + getCompleteAddress(address: AztecAddress): Promise { + return Promise.resolve(this.#getCompleteAddress(address)); } getCompleteAddresses(): Promise { @@ -343,7 +354,7 @@ export class KVPxeDatabase implements PxeDatabase { } estimateSize(): number { - const notesSize = Array.from(this.#getAllNonNullifiedNotes()).reduce((sum, note) => sum + note.getSize(), 0); + const notesSize = Array.from(this.#getNotes({})).reduce((sum, note) => sum + note.getSize(), 0); const authWitsSize = Array.from(this.#authWitnesses.values()).reduce( (sum, value) => sum + value.length * Fr.SIZE_IN_BYTES, 0, From 90a4dc47d3248789d05c88ceda3cfe26af2d9645 Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 17 Jan 2024 12:46:07 +0000 Subject: [PATCH 25/50] multi-jsonRPC server fixes --- .../src/json-rpc/client/json_rpc_client.ts | 4 ++-- .../foundation/src/json-rpc/server/json_proxy.ts | 13 ++++++++++--- .../src/json-rpc/server/json_rpc_server.ts | 9 +++++---- yarn-project/pxe/src/synchronizer/synchronizer.ts | 3 --- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts b/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts index ea4f2258061..bb2c724c49e 100644 --- a/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts +++ b/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts @@ -133,8 +133,8 @@ export function createJsonRpcClient( if (namespaceMethods) { rpcMethod = `${namespaceMethods}_${method}`; } - if (['then', 'catch'].includes(rpcMethod)) { - return Reflect.get(target, rpcMethod); + if (['then', 'catch'].includes(method)) { + return Reflect.get(target, method); } return (...params: any[]) => { debug(format(`JsonRpcClient.constructor`, 'proxy', rpcMethod, '<-', params)); diff --git a/yarn-project/foundation/src/json-rpc/server/json_proxy.ts b/yarn-project/foundation/src/json-rpc/server/json_proxy.ts index bad680e04d4..4cea1603aa9 100644 --- a/yarn-project/foundation/src/json-rpc/server/json_proxy.ts +++ b/yarn-project/foundation/src/json-rpc/server/json_proxy.ts @@ -7,6 +7,9 @@ import { assert, hasOwnProperty } from '../js_utils.js'; const debug = createDebugLogger('json-rpc:json_proxy'); +/** + * A map of class names to class constructors. + */ export type ClassMaps = { /** The String class map */ stringClassMap: StringClassConverterInput; @@ -31,16 +34,20 @@ export class JsonProxy { * Call an RPC method. * @param methodName - The RPC method. * @param jsonParams - The RPG parameters. + * @param skipConversion - Whether to skip conversion of the parameters. * @returns The remote result. */ - public async call(methodName: string, jsonParams: any[] = []) { + public async call(methodName: string, jsonParams: any[] = [], skipConversion = false) { debug(format(`JsonProxy:call`, methodName, jsonParams)); // Get access to our class members const proto = Object.getPrototypeOf(this.handler); assert(hasOwnProperty(proto, methodName), `JsonProxy: Method ${methodName} not found!`); - assert(Array.isArray(jsonParams), 'JsonProxy: Params not an array!'); + assert(Array.isArray(jsonParams), `JsonProxy: ${methodName} params not an array: ${jsonParams}`); // convert the params from json representation to classes - const convertedParams = jsonParams.map(param => convertFromJsonObj(this.classConverter, param)); + let convertedParams = jsonParams; + if (!skipConversion) { + convertedParams = jsonParams.map(param => convertFromJsonObj(this.classConverter, param)); + } debug(format('JsonProxy:call', methodName, '<-', convertedParams)); const rawRet = await (this.handler as any)[methodName](...convertedParams); const ret = convertToJsonObj(this.classConverter, rawRet); diff --git a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts index 81a44e01002..c1fc3a59d49 100644 --- a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts +++ b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts @@ -212,10 +212,11 @@ export class JsonRpcServer { * Call an RPC method. * @param methodName - The RPC method. * @param jsonParams - The RPG parameters. + * @param skipConversion - Whether to skip conversion of the parameters. * @returns The remote result. */ - public async call(methodName: string, jsonParams: any[] = []) { - return await this.proxy.call(methodName, jsonParams); + public async call(methodName: string, jsonParams: any[] = [], skipConversion: boolean) { + return await this.proxy.call(methodName, jsonParams, skipConversion); } } @@ -282,7 +283,7 @@ export function createMultiJsonRpcServer( const namespacedMethod = `${namespace}_${method}`; handler[namespacedMethod] = (...args: any[]) => { - return server.call(method, ...args); + return server.call(method, args, true); }; } @@ -307,5 +308,5 @@ export function createMultiJsonRpcServer( { stringClassMap: {}, objectClassMap: {} } as ClassMaps, ); - return new JsonRpcServer(handler, classMaps.stringClassMap, classMaps.objectClassMap, false, [], log); + return new JsonRpcServer(Object.create(handler), classMaps.stringClassMap, classMaps.objectClassMap, false, [], log); } diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.ts b/yarn-project/pxe/src/synchronizer/synchronizer.ts index e3fdfe2b264..d0088fd9cc4 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.ts @@ -148,14 +148,11 @@ export class Synchronizer { const logCount = L2BlockL2Logs.getTotalLogCount(encryptedLogs); this.log(`Forwarding ${logCount} encrypted logs and blocks to ${this.noteProcessors.length} note processors`); - let i = 1; for (const noteProcessor of this.noteProcessors) { - console.log('processing note processor', i++); await noteProcessor.process(blockContexts, encryptedLogs); } return true; } catch (err) { - console.log(err); this.log.error(`Error in synchronizer work`, err); return false; } From 1c04047df5a2b767672addf5bfa74e899bad32b0 Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 17 Jan 2024 14:36:59 +0000 Subject: [PATCH 26/50] export types --- yarn-project/aztec/src/cli/cli.ts | 33 ++++++++++--------- .../src/json-rpc/client/json_rpc_client.ts | 1 + .../foundation/src/json-rpc/server/index.ts | 2 +- .../src/json-rpc/server/json_rpc_server.ts | 2 +- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index d320eabe835..c60aab82f28 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -4,7 +4,7 @@ import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeC import { AccountManager, GrumpkinScalar, fileURLToPath } from '@aztec/aztec.js'; import { createAztecNodeClient } from '@aztec/circuit-types'; import { NULL_KEY } from '@aztec/ethereum'; -import { createMultiJsonRpcServer, startHttpRpcServer } from '@aztec/foundation/json-rpc/server'; +import { ServerList, createMultiJsonRpcServer, startHttpRpcServer } from '@aztec/foundation/json-rpc/server'; import { DebugLogger, LogFn } from '@aztec/foundation/log'; import { BootstrapNode, P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; import { PXEService, PXEServiceConfig, createPXERpcServer, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; @@ -23,7 +23,6 @@ import { openDb } from './util.js'; const { AZTEC_PORT = '8080', - AZTEC_NODE_PORT = '8079', PXE_PORT = '8080', ARCHIVER_PORT = '8080', AZTEC_NODE_URL, @@ -128,11 +127,6 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { const httpServer = http.createServer(app.callback()); httpServer.listen(options.port); userLog(`Aztec Server listening on port ${options.port}`); - - // startHttpRpcServer(node, createAztecNodeRpcServer, options.nodePort); - // userLog(`Aztec Node JSON-RPC Server listening on nodePort ${options.nodePort}`); - // startHttpRpcServer(pxe, createPXERpcServer, options.pxePort); - // userLog(`PXE JSON-RPC Server listening on port ${options.pxePort}`); }); // Start Aztec modules with options @@ -152,6 +146,8 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { // Start Aztec Node if (options.node) { + // Services that will be started in a single multi-rpc server + const services: ServerList = []; // get env vars first const aztecNodeConfigEnvVars = getNodeConfigEnvVars(); // get config from options @@ -194,20 +190,27 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { // Create and start Aztec Node. const node = await createAztecNode(nodeConfig); - startHttpRpcServer(node, createAztecNodeRpcServer, nodeCliOptions.port || AZTEC_NODE_PORT); - userLog(`Aztec Node JSON-RPC Server listening on port ${nodeCliOptions.port || AZTEC_NODE_PORT}`); + const nodeServer = createAztecNodeRpcServer(node); + + // Add node to services list + services.push({ node: nodeServer }); + + // Add node stop function to signal handlers + signalHandlers.push(node.stop); // Create a PXE client that connects to the node. if (options.pxe) { const pxeCliOptions = parseModuleOptions(options.pxe); - const pxe = await createAztecPXE(node); - signalHandlers.push(pxe.stop); + const pxeConfig = mergeEnvVarsAndCliOptions(getPXEServiceConfig(), pxeCliOptions); + const pxe = await createAztecPXE(node, pxeConfig); + const pxeServer = createPXERpcServer(pxe); - // Start PXE JSON-RPC server. - startHttpRpcServer(pxe, createPXERpcServer, pxeCliOptions.port || PXE_PORT); - } + // Add PXE to services list + services.push({ pxe: pxeServer }); - signalHandlers.push(node.stop); + // Add PXE stop function to signal handlers + signalHandlers.push(pxe.stop); + } } else if (options.pxe) { // Starting a PXE with a remote node. // get env vars first diff --git a/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts b/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts index bb2c724c49e..7d947c6b5e5 100644 --- a/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts +++ b/yarn-project/foundation/src/json-rpc/client/json_rpc_client.ts @@ -92,6 +92,7 @@ export function makeFetch(retries: number[], noRetry: boolean, log?: DebugLogger * @param stringClassMap - A map of class names to string representations. * @param objectClassMap - A map of class names to class constructors. * @param useApiEndpoints - Whether to use the API endpoints or the default RPC endpoint. + * @param namespaceMethods - String value (or false/empty) to namespace all methods sent to the server. e.g. 'getInfo' -\> 'pxe_getInfo' * @param fetch - The fetch implementation to use. */ export function createJsonRpcClient( diff --git a/yarn-project/foundation/src/json-rpc/server/index.ts b/yarn-project/foundation/src/json-rpc/server/index.ts index 7937ba0e703..a0f5caf72ca 100644 --- a/yarn-project/foundation/src/json-rpc/server/index.ts +++ b/yarn-project/foundation/src/json-rpc/server/index.ts @@ -1,2 +1,2 @@ -export { JsonRpcServer, createStatusRouter, startHttpRpcServer, createMultiJsonRpcServer } from './json_rpc_server.js'; +export * from './json_rpc_server.js'; export { JsonProxy } from './json_proxy.js'; diff --git a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts index c1fc3a59d49..c2d6ff6f77a 100644 --- a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts +++ b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts @@ -257,7 +257,7 @@ export function startHttpRpcServer( /** * List of namespace to server instance. */ -type ServerList = { +export type ServerList = { /** name of the service to be used for namespacing */ [name: string]: JsonRpcServer; }[]; From 07d890a9c86762c82f280f36a2bca07c9d25ffd6 Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 17 Jan 2024 17:37:20 +0000 Subject: [PATCH 27/50] update tests --- yarn-project/aztec/src/cli/cli.ts | 38 ++++++++++--------- .../src/aztec_node/rpc/aztec_node_client.ts | 2 +- .../scripts/docker-compose-browser.yml | 1 + .../end-to-end/scripts/docker-compose-p2p.yml | 1 - .../end-to-end/scripts/docker-compose.yml | 5 +-- yarn-project/end-to-end/src/fixtures/utils.ts | 23 ++++------- .../p2p/src/tx_pool/aztec_kv_tx_pool.ts | 2 +- 7 files changed, 33 insertions(+), 39 deletions(-) diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index c60aab82f28..b74b0897ada 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -21,13 +21,7 @@ import { github, splash } from '../splash.js'; import { cliTexts } from './texts.js'; import { openDb } from './util.js'; -const { - AZTEC_PORT = '8080', - PXE_PORT = '8080', - ARCHIVER_PORT = '8080', - AZTEC_NODE_URL, - DEPLOY_AZTEC_CONTRACTS, -} = process.env; +const { AZTEC_PORT = '8080', AZTEC_NODE_URL, DEPLOY_AZTEC_CONTRACTS } = process.env; const installSignalHandlers = (logFn: LogFn, cb?: Array<() => Promise>) => { const shutdown = async () => { @@ -96,10 +90,7 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { .command('sandbox') .description('Starts Aztec sandbox.') .option('-p, --port ', 'Port to run Aztec on.', AZTEC_PORT) - // .option('-np, --node-port ', 'Port to run Aztec Node on.', AZTEC_NODE_PORT) - // .option('-pp, --pxe-port ', 'Port to run PXE on (optional).', PXE_PORT) .option('-s, --skip-test-accounts', 'DO NOT deploy test accounts.', false) - // .option() .action(async options => { userLog(`${splash}\n${github}\n\n`); userLog(`Setting up Aztec Sandbox v${cliVersion}, please stand by...`); @@ -135,19 +126,20 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { .description( 'Starts Aztec modules. Options for each module can be set as key-value pairs (e.g. "option1=value1,option2=value2") or as environment variables.', ) + .option('-p, --port ', 'Port to run Aztec on.', AZTEC_PORT) .option('-n, --node [options]', cliTexts.node) - .option('-p, --pxe [options]', cliTexts.pxe) + .option('-px, --pxe [options]', cliTexts.pxe) .option('-a, --archiver [options]', cliTexts.archiver) .option('-s, --sequencer [options]', cliTexts.sequencer) - .option('-p2p, --p2p-bootstrap', cliTexts.p2pBootstrap) + .option('-p2p, --p2p-bootstrap [options]', cliTexts.p2pBootstrap) .action(async options => { + // Services that will be started in a single multi-rpc server + const services: ServerList = []; // list of 'stop' functions to call when process ends const signalHandlers: Array<() => Promise> = []; // Start Aztec Node if (options.node) { - // Services that will be started in a single multi-rpc server - const services: ServerList = []; // get env vars first const aztecNodeConfigEnvVars = getNodeConfigEnvVars(); // get config from options @@ -160,6 +152,7 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { // expect archiver url in node config const archiverUrl = nodeCliOptions.archiverUrl; if (!archiverUrl) { + userLog('Archiver Service URL is required to start Aztec Node without --archiver option'); throw new Error('Archiver Service URL is required to start Aztec Node without --archiver option'); } nodeConfig.archiverUrl = archiverUrl; @@ -234,10 +227,9 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { const node = createAztecNodeClient(nodeUrl); const pxe = await createPXEService(node, pxeConfig); + const pxeServer = createPXERpcServer(pxe); + services.push({ pxe: pxeServer }); signalHandlers.push(pxe.stop); - // Start PXE JSON-RPC server - startHttpRpcServer(pxe, createPXERpcServer, pxeCliOptions.port || PXE_PORT); - userLog(`PXE JSON-RPC Server listening on port ${pxeCliOptions.port || PXE_PORT}`); } else if (options.archiver) { // Start a standalone archiver. // get env vars first @@ -251,7 +243,9 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { const archiverStore = new LMDBArchiverStore(nodeDb, archiverConfig.maxLogs); const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, true); - startHttpRpcServer(archiver, createArchiverRpcServer, archiverCliOptions.port || ARCHIVER_PORT); + const archiverServer = createArchiverRpcServer(archiver); + services.push({ archiver: archiverServer }); + signalHandlers.push(archiver.stop); } else if (options.p2pBootstrap) { // Start a P2P bootstrap node. const envVars = getP2PConfigEnvVars(); @@ -261,6 +255,14 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { await bootstrapNode.start(config); signalHandlers.push(bootstrapNode.stop); } + if (services.length) { + const rpcServer = createMultiJsonRpcServer(services, debugLogger); + + const app = rpcServer.getApp(); + const httpServer = http.createServer(app.callback()); + httpServer.listen(options.port); + userLog(`Aztec Server listening on port ${options.port}`); + } installSignalHandlers(debugLogger, signalHandlers); }); return program; diff --git a/yarn-project/circuit-types/src/aztec_node/rpc/aztec_node_client.ts b/yarn-project/circuit-types/src/aztec_node/rpc/aztec_node_client.ts index 0ea76ae221d..32a4dd48d97 100644 --- a/yarn-project/circuit-types/src/aztec_node/rpc/aztec_node_client.ts +++ b/yarn-project/circuit-types/src/aztec_node/rpc/aztec_node_client.ts @@ -42,7 +42,7 @@ export function createAztecNodeClient(url: string, fetch = defaultFetch): AztecN }, { Tx, L2BlockL2Logs }, false, - false, + 'node', fetch, ); } diff --git a/yarn-project/end-to-end/scripts/docker-compose-browser.yml b/yarn-project/end-to-end/scripts/docker-compose-browser.yml index c812613848b..42df45c8163 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-browser.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-browser.yml @@ -38,6 +38,7 @@ services: ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 PXE_URL: http://sandbox:8080 + AZTEC_URL: http://sandbox:8080 entrypoint: ['./scripts/start_e2e_ci_browser.sh', './src/e2e_aztec_js_browser.test.ts'] volumes: - ../log:/usr/src/yarn-project/end-to-end/log:rw diff --git a/yarn-project/end-to-end/scripts/docker-compose-p2p.yml b/yarn-project/end-to-end/scripts/docker-compose-p2p.yml index b7f9073ba7b..50cca540d6c 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-p2p.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-p2p.yml @@ -11,7 +11,6 @@ services: command: 'start --p2p-bootstrap' ports: - '40400:40400' - command: 'start' environment: DEBUG: 'aztec:*' DEBUG_COLORS: 1 diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 8b00ebff566..65c6c313325 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -16,8 +16,7 @@ services: image: aztecprotocol/aztec:latest command: sandbox environment: - DEBUG: 'aztec:*' - PXE_PORT: 8080 + DEBUG: 'aztec:*,json-rpc:*' DEBUG_COLORS: 1 ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 @@ -29,7 +28,6 @@ services: ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 ports: - '8080:8080' - - '8079:8079' end-to-end: image: aztecprotocol/end-to-end:latest @@ -39,6 +37,7 @@ services: DEBUG_COLORS: 1 ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 + AZTEC_URL: http://sandbox:8080 PXE_URL: http://sandbox:8080 command: ${TEST:-./src/e2e_deploy_contract.test.ts} volumes: diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index 64ebe0fdf0a..12261ee101b 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -55,17 +55,10 @@ import { isMetricsLoggingRequested, setupMetricsLogger } from './logging.js'; export { deployAndInitializeTokenAndBridgeContracts } from '../shared/cross_chain_test_harness.js'; -const { PXE_URL = '', AZTEC_NODE_URL = '' } = process.env; +const { AZTEC_URL = 'http://localhost:8080' } = process.env; -const getAztecNodeUrl = () => { - if (AZTEC_NODE_URL) { - return AZTEC_NODE_URL; - } - - // If AZTEC_NODE_URL is not set, we assume that the PXE is running on the same host as the Aztec Node and use the default port - const url = new URL(PXE_URL); - url.port = '8079'; - return url.toString(); +const getAztecUrl = () => { + return AZTEC_URL; }; export const setupL1Contracts = async ( @@ -168,14 +161,14 @@ async function setupWithRemoteEnvironment( numberOfAccounts: number, ) { // we are setting up against a remote environment, l1 contracts are already deployed - const aztecNodeUrl = getAztecNodeUrl(); + const aztecNodeUrl = getAztecUrl(); logger(`Creating Aztec Node client to remote host ${aztecNodeUrl}`); const aztecNode = createAztecNodeClient(aztecNodeUrl); - logger(`Creating PXE client to remote host ${PXE_URL}`); - const pxeClient = createPXEClient(PXE_URL); + logger(`Creating PXE client to remote host ${AZTEC_URL}`); + const pxeClient = createPXEClient(AZTEC_URL); await waitForPXE(pxeClient, logger); logger('JSON RPC client connected to PXE'); - logger(`Retrieving contract addresses from ${PXE_URL}`); + logger(`Retrieving contract addresses from ${AZTEC_URL}`); const l1Contracts = (await pxeClient.getNodeInfo()).l1ContractAddresses; logger('PXE created, constructing available wallets from already registered accounts...'); const wallets = await getDeployedTestAccountsWallets(pxeClient); @@ -280,7 +273,7 @@ export async function setup( const privKeyRaw = hdAccount.getHdKey().privateKey; const publisherPrivKey = privKeyRaw === null ? null : Buffer.from(privKeyRaw); - if (PXE_URL) { + if (AZTEC_URL) { // we are setting up against a remote environment, l1 contracts are assumed to already be deployed return await setupWithRemoteEnvironment(hdAccount, config, logger, numberOfAccounts); } diff --git a/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts b/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts index 1d8d723ab5d..54273a45387 100644 --- a/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts +++ b/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts @@ -49,7 +49,7 @@ export class AztecKVTxPool implements TxPool { return this.#store.transaction(() => { for (const [i, tx] of txs.entries()) { const txHash = txHashes[i]; - this.#log.info(`Adding tx with id ${txHash.toString()}`, { + this.#log.info(`FOOBAR Adding tx with id ${txHash.toString()}`, { eventName: 'tx-added-to-pool', ...tx.getStats(), } satisfies TxAddedToPoolStats); From d924b25a4ce2866a1873bce561edfd91dd1cd105 Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 17 Jan 2024 17:45:57 +0000 Subject: [PATCH 28/50] fix formatting --- yarn-project/aztec/src/cli/cli.ts | 2 +- .../end-to-end/scripts/docker-compose-browser.yml | 1 - yarn-project/end-to-end/scripts/docker-compose.yml | 1 - yarn-project/end-to-end/src/fixtures/utils.ts | 12 ++++++------ 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index b74b0897ada..1c390b944db 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -4,7 +4,7 @@ import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeC import { AccountManager, GrumpkinScalar, fileURLToPath } from '@aztec/aztec.js'; import { createAztecNodeClient } from '@aztec/circuit-types'; import { NULL_KEY } from '@aztec/ethereum'; -import { ServerList, createMultiJsonRpcServer, startHttpRpcServer } from '@aztec/foundation/json-rpc/server'; +import { ServerList, createMultiJsonRpcServer } from '@aztec/foundation/json-rpc/server'; import { DebugLogger, LogFn } from '@aztec/foundation/log'; import { BootstrapNode, P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; import { PXEService, PXEServiceConfig, createPXERpcServer, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; diff --git a/yarn-project/end-to-end/scripts/docker-compose-browser.yml b/yarn-project/end-to-end/scripts/docker-compose-browser.yml index 42df45c8163..c812613848b 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-browser.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-browser.yml @@ -38,7 +38,6 @@ services: ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 PXE_URL: http://sandbox:8080 - AZTEC_URL: http://sandbox:8080 entrypoint: ['./scripts/start_e2e_ci_browser.sh', './src/e2e_aztec_js_browser.test.ts'] volumes: - ../log:/usr/src/yarn-project/end-to-end/log:rw diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 65c6c313325..9f3cfc3650d 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -37,7 +37,6 @@ services: DEBUG_COLORS: 1 ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 - AZTEC_URL: http://sandbox:8080 PXE_URL: http://sandbox:8080 command: ${TEST:-./src/e2e_deploy_contract.test.ts} volumes: diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index 12261ee101b..b1b10e5a0ea 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -55,10 +55,10 @@ import { isMetricsLoggingRequested, setupMetricsLogger } from './logging.js'; export { deployAndInitializeTokenAndBridgeContracts } from '../shared/cross_chain_test_harness.js'; -const { AZTEC_URL = 'http://localhost:8080' } = process.env; +const { PXE_URL = 'http://localhost:8080' } = process.env; const getAztecUrl = () => { - return AZTEC_URL; + return PXE_URL; }; export const setupL1Contracts = async ( @@ -164,11 +164,11 @@ async function setupWithRemoteEnvironment( const aztecNodeUrl = getAztecUrl(); logger(`Creating Aztec Node client to remote host ${aztecNodeUrl}`); const aztecNode = createAztecNodeClient(aztecNodeUrl); - logger(`Creating PXE client to remote host ${AZTEC_URL}`); - const pxeClient = createPXEClient(AZTEC_URL); + logger(`Creating PXE client to remote host ${PXE_URL}`); + const pxeClient = createPXEClient(PXE_URL); await waitForPXE(pxeClient, logger); logger('JSON RPC client connected to PXE'); - logger(`Retrieving contract addresses from ${AZTEC_URL}`); + logger(`Retrieving contract addresses from ${PXE_URL}`); const l1Contracts = (await pxeClient.getNodeInfo()).l1ContractAddresses; logger('PXE created, constructing available wallets from already registered accounts...'); const wallets = await getDeployedTestAccountsWallets(pxeClient); @@ -273,7 +273,7 @@ export async function setup( const privKeyRaw = hdAccount.getHdKey().privateKey; const publisherPrivKey = privKeyRaw === null ? null : Buffer.from(privKeyRaw); - if (AZTEC_URL) { + if (PXE_URL) { // we are setting up against a remote environment, l1 contracts are assumed to already be deployed return await setupWithRemoteEnvironment(hdAccount, config, logger, numberOfAccounts); } From 6d357ebb1a48505f0fedae3e87b276ac51a7c4ba Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 17 Jan 2024 17:52:54 +0000 Subject: [PATCH 29/50] remove json-rpc logs --- aztec-up/bin/docker-compose.yml | 2 -- yarn-project/aztec/docker-compose.yml | 1 - yarn-project/end-to-end/scripts/docker-compose.yml | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/aztec-up/bin/docker-compose.yml b/aztec-up/bin/docker-compose.yml index 6716ca6415b..425a9be9636 100644 --- a/aztec-up/bin/docker-compose.yml +++ b/aztec-up/bin/docker-compose.yml @@ -20,7 +20,6 @@ services: image: "aztecprotocol/aztec" command: sandbox ports: - - "${AZTEC_NODE_PORT:-8079}:${AZTEC_NODE_PORT:-8079}" - "${PXE_PORT:-8080}:${PXE_PORT:-8080}" environment: DEBUG: # Loaded from the user shell if explicitly set @@ -33,7 +32,6 @@ services: WS_BLOCK_CHECK_INTERVAL_MS: 50 PXE_BLOCK_POLLING_INTERVAL_MS: 50 ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 - AZTEC_NODE_PORT: ${AZTEC_NODE_PORT:-8079} PXE_PORT: ${PXE_PORT:-8080} volumes: - ./log:/usr/src/yarn-project/aztec/log:rw diff --git a/yarn-project/aztec/docker-compose.yml b/yarn-project/aztec/docker-compose.yml index 04ec4e7d9bc..84397ba70b7 100644 --- a/yarn-project/aztec/docker-compose.yml +++ b/yarn-project/aztec/docker-compose.yml @@ -17,7 +17,6 @@ services: aztec: image: 'aztecprotocol/aztec:${SANDBOX_VERSION:-latest}' ports: - - '${SANDBOX_AZTEC_NODE_PORT:-8079}:8079' - '${SANDBOX_PXE_PORT:-8080}:8080' environment: DEBUG: # Loaded from the user shell if explicitly set diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 9f3cfc3650d..8758c8a306c 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -16,7 +16,7 @@ services: image: aztecprotocol/aztec:latest command: sandbox environment: - DEBUG: 'aztec:*,json-rpc:*' + DEBUG: 'aztec:*' DEBUG_COLORS: 1 ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 From 7245c724294531ae445570a75e841d597c6417cf Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 17 Jan 2024 22:22:27 +0000 Subject: [PATCH 30/50] fix: notes database --- yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts | 2 +- yarn-project/pxe/src/database/kv_pxe_database.ts | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts b/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts index 54273a45387..1d8d723ab5d 100644 --- a/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts +++ b/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts @@ -49,7 +49,7 @@ export class AztecKVTxPool implements TxPool { return this.#store.transaction(() => { for (const [i, tx] of txs.entries()) { const txHash = txHashes[i]; - this.#log.info(`FOOBAR Adding tx with id ${txHash.toString()}`, { + this.#log.info(`Adding tx with id ${txHash.toString()}`, { eventName: 'tx-added-to-pool', ...tx.getStats(), } satisfies TxAddedToPoolStats); diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index a8f1bb47ac5..c58945d4da8 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -30,7 +30,7 @@ export class KVPxeDatabase implements PxeDatabase { #contracts: AztecMap; #notes: AztecMap; - #notesByNullifier: AztecMap; + #nullifierToNoteId: AztecMap; #notesByContract: AztecMultiMap; #notesByStorageSlot: AztecMultiMap; #notesByTxHash: AztecMultiMap; @@ -56,7 +56,7 @@ export class KVPxeDatabase implements PxeDatabase { this.#syncedBlockPerPublicKey = db.createMap('synced_block_per_public_key'); this.#notes = db.createMap('notes'); - this.#notesByNullifier = db.createMap('notes'); + this.#nullifierToNoteId = db.createMap('nullifier_to_note'); this.#notesByContract = db.createMultiMap('notes_by_contract'); this.#notesByStorageSlot = db.createMultiMap('notes_by_storage_slot'); this.#notesByTxHash = db.createMultiMap('notes_by_tx_hash'); @@ -98,9 +98,9 @@ export class KVPxeDatabase implements PxeDatabase { // this provides the uniqueness we need to store individual notes // and should also return notes in the order that they were created. // Had we stored them by their nullifier, they would be returned in random order - const noteIndex = toBufferBE(dao.index, 32).toString(); + const noteIndex = toBufferBE(dao.index, 32).toString('hex'); void this.#notes.set(noteIndex, dao.toBuffer()); - void this.#notesByNullifier.set(dao.siloedNullifier.toString(), noteIndex); + void this.#nullifierToNoteId.set(dao.siloedNullifier.toString(), noteIndex); void this.#notesByContract.set(dao.contractAddress.toString(), noteIndex); void this.#notesByStorageSlot.set(dao.storageSlot.toString(), noteIndex); void this.#notesByTxHash.set(dao.txHash.toString(), noteIndex); @@ -217,7 +217,7 @@ export class KVPxeDatabase implements PxeDatabase { const nullifiedNotes: NoteDao[] = []; for (const nullifier of nullifiers) { - const noteIndex = this.#notesByNullifier.get(nullifier.toString()); + const noteIndex = this.#nullifierToNoteId.get(nullifier.toString()); if (!noteIndex) { continue; } @@ -237,10 +237,13 @@ export class KVPxeDatabase implements PxeDatabase { nullifiedNotes.push(note); + void this.#notes.delete(noteIndex); void this.#notesByOwner.deleteValue(account.toString(), noteIndex); void this.#notesByTxHash.deleteValue(note.txHash.toString(), noteIndex); void this.#notesByContract.deleteValue(note.contractAddress.toString(), noteIndex); void this.#notesByStorageSlot.deleteValue(note.storageSlot.toString(), noteIndex); + + void this.#nullifierToNoteId.delete(nullifier.toString()); } return nullifiedNotes; From c7e29f507c41b1c61e4c8a8533aadca48394bb91 Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 18 Jan 2024 09:57:55 +0000 Subject: [PATCH 31/50] skip return conversion as well in multi rpc --- yarn-project/end-to-end/package.json | 3 ++- yarn-project/end-to-end/src/fixtures/utils.ts | 2 +- yarn-project/foundation/src/json-rpc/server/json_proxy.ts | 5 ++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/yarn-project/end-to-end/package.json b/yarn-project/end-to-end/package.json index f0d25e2a6db..4456483d851 100644 --- a/yarn-project/end-to-end/package.json +++ b/yarn-project/end-to-end/package.json @@ -5,6 +5,7 @@ "exports": "./dest/index.js", "scripts": { "build": "yarn clean && tsc -b && webpack", + "build:e2e": "yarn clean && tsc -b", "build:dev": "tsc -b --watch", "build:web": "webpack", "clean": "rm -rf ./dest .tsbuildinfo", @@ -89,4 +90,4 @@ "!*.test.*" ], "types": "./dest/index.d.ts" -} +} \ No newline at end of file diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index b1b10e5a0ea..0bce7a88f52 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -55,7 +55,7 @@ import { isMetricsLoggingRequested, setupMetricsLogger } from './logging.js'; export { deployAndInitializeTokenAndBridgeContracts } from '../shared/cross_chain_test_harness.js'; -const { PXE_URL = 'http://localhost:8080' } = process.env; +const { PXE_URL = '' } = process.env; const getAztecUrl = () => { return PXE_URL; diff --git a/yarn-project/foundation/src/json-rpc/server/json_proxy.ts b/yarn-project/foundation/src/json-rpc/server/json_proxy.ts index 4cea1603aa9..af56d4ce932 100644 --- a/yarn-project/foundation/src/json-rpc/server/json_proxy.ts +++ b/yarn-project/foundation/src/json-rpc/server/json_proxy.ts @@ -50,7 +50,10 @@ export class JsonProxy { } debug(format('JsonProxy:call', methodName, '<-', convertedParams)); const rawRet = await (this.handler as any)[methodName](...convertedParams); - const ret = convertToJsonObj(this.classConverter, rawRet); + let ret = rawRet; + if (skipConversion) { + ret = convertToJsonObj(this.classConverter, rawRet); + } debug(format('JsonProxy:call', methodName, '->', ret)); return ret; } From 40288a75a935895b492a6814b6454638f633994d Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 18 Jan 2024 11:32:12 +0000 Subject: [PATCH 32/50] update boxes yarn.lock --- boxes/yarn.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/boxes/yarn.lock b/boxes/yarn.lock index 5719e430a8d..0f764e1dd06 100644 --- a/boxes/yarn.lock +++ b/boxes/yarn.lock @@ -302,6 +302,7 @@ __metadata: resolution: "@aztec/types@portal:../yarn-project/types::locator=%40aztec%2Fboxes%40workspace%3A." dependencies: "@aztec/ethereum": "workspace:^" + "@aztec/foundation": "workspace:^" languageName: node linkType: soft From 556c5de152e561a106122495b63285c274a93f90 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 18 Jan 2024 11:33:38 +0000 Subject: [PATCH 33/50] fix: give mock Notes an index --- yarn-project/pxe/src/database/note_dao.test.ts | 2 +- yarn-project/pxe/src/database/pxe_database_test_suite.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/yarn-project/pxe/src/database/note_dao.test.ts b/yarn-project/pxe/src/database/note_dao.test.ts index 34baa9a14dd..113f2de5fba 100644 --- a/yarn-project/pxe/src/database/note_dao.test.ts +++ b/yarn-project/pxe/src/database/note_dao.test.ts @@ -11,7 +11,7 @@ export const randomNoteDao = ({ nonce = Fr.random(), innerNoteHash = Fr.random(), siloedNullifier = Fr.random(), - index = BigInt(0), + index = Fr.random().toBigInt(), publicKey = Point.random(), }: Partial = {}) => { return new NoteDao( diff --git a/yarn-project/pxe/src/database/pxe_database_test_suite.ts b/yarn-project/pxe/src/database/pxe_database_test_suite.ts index 5464ef98dde..0125e62b2f9 100644 --- a/yarn-project/pxe/src/database/pxe_database_test_suite.ts +++ b/yarn-project/pxe/src/database/pxe_database_test_suite.ts @@ -110,6 +110,7 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { contractAddress: contractAddresses[i % contractAddresses.length], storageSlot: storageSlots[i % storageSlots.length], publicKey: owners[i % owners.length].publicKey, + index: BigInt(i), }), ); }); From 05ec40d84f72de531c5bd5ee150122d09563207b Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 18 Jan 2024 12:40:45 +0000 Subject: [PATCH 34/50] fix blank box webpack --- boxes/blank/webpack.config.js | 1 + 1 file changed, 1 insertion(+) diff --git a/boxes/blank/webpack.config.js b/boxes/blank/webpack.config.js index abbbfa719b5..606e154a4f8 100644 --- a/boxes/blank/webpack.config.js +++ b/boxes/blank/webpack.config.js @@ -56,6 +56,7 @@ export default (_, argv) => ({ crypto: false, fs: false, path: false, + events: require.resolve('events/'), stream: require.resolve('stream-browserify'), tty: require.resolve('tty-browserify'), util: require.resolve('util/'), From 6f127840377c1397e9a4e242fe28f60b1eb0f840 Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 18 Jan 2024 13:09:17 +0000 Subject: [PATCH 35/50] fix json-rpc conversion --- yarn-project/foundation/src/json-rpc/server/json_proxy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/foundation/src/json-rpc/server/json_proxy.ts b/yarn-project/foundation/src/json-rpc/server/json_proxy.ts index af56d4ce932..b46a1983ada 100644 --- a/yarn-project/foundation/src/json-rpc/server/json_proxy.ts +++ b/yarn-project/foundation/src/json-rpc/server/json_proxy.ts @@ -51,7 +51,7 @@ export class JsonProxy { debug(format('JsonProxy:call', methodName, '<-', convertedParams)); const rawRet = await (this.handler as any)[methodName](...convertedParams); let ret = rawRet; - if (skipConversion) { + if (!skipConversion) { ret = convertToJsonObj(this.classConverter, rawRet); } debug(format('JsonProxy:call', methodName, '->', ret)); From eb83d0fdc9a07d5b510aa8e2bd5e4a749fbe4d2b Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 19 Jan 2024 13:42:49 +0000 Subject: [PATCH 36/50] update terraforms --- yarn-project/aztec-node/terraform/main.tf | 1 + yarn-project/p2p-bootstrap/terraform/main.tf | 1 + 2 files changed, 2 insertions(+) diff --git a/yarn-project/aztec-node/terraform/main.tf b/yarn-project/aztec-node/terraform/main.tf index a65230f6a16..6b505ab1ee9 100644 --- a/yarn-project/aztec-node/terraform/main.tf +++ b/yarn-project/aztec-node/terraform/main.tf @@ -155,6 +155,7 @@ resource "aws_ecs_task_definition" "aztec-node" { { "name": "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}", "image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}", + "command": ["sandbox"], "essential": true, "memoryReservation": 3776, "portMappings": [ diff --git a/yarn-project/p2p-bootstrap/terraform/main.tf b/yarn-project/p2p-bootstrap/terraform/main.tf index 8245a787bc5..376d59c134b 100644 --- a/yarn-project/p2p-bootstrap/terraform/main.tf +++ b/yarn-project/p2p-bootstrap/terraform/main.tf @@ -106,6 +106,7 @@ resource "aws_ecs_task_definition" "p2p-bootstrap" { { "name": "${var.DEPLOY_TAG}-p2p-bootstrap-${count.index + 1}", "image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}", + "command": ["start", "--p2p-bootstrap"], "essential": true, "command": ["start"], "memoryReservation": 3776, From 04a45b32786c1649fef0113418dfb48f55baa10a Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 19 Jan 2024 14:51:11 +0000 Subject: [PATCH 37/50] remove openDb util --- yarn-project/aztec/package.json | 1 + yarn-project/aztec/src/cli/cli.ts | 11 +++-- yarn-project/aztec/src/cli/util.ts | 78 ------------------------------ yarn-project/yarn.lock | 1 + 4 files changed, 9 insertions(+), 82 deletions(-) delete mode 100644 yarn-project/aztec/src/cli/util.ts diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index 732bb4c5fd8..98163679e41 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -35,6 +35,7 @@ "@aztec/circuits.js": "workspace:^", "@aztec/ethereum": "workspace:^", "@aztec/foundation": "workspace:^", + "@aztec/kv-store": "workspace:^", "@aztec/l1-artifacts": "workspace:^", "@aztec/noir-compiler": "workspace:^", "@aztec/noir-contracts": "workspace:^", diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index 1c390b944db..19eedab36f3 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -1,11 +1,12 @@ import { deployInitialTestAccounts } from '@aztec/accounts/testing'; -import { Archiver, LMDBArchiverStore, createArchiverRpcServer, getConfigEnvVars } from '@aztec/archiver'; +import { Archiver, KVArchiverDataStore, createArchiverRpcServer, getConfigEnvVars } from '@aztec/archiver'; import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; import { AccountManager, GrumpkinScalar, fileURLToPath } from '@aztec/aztec.js'; import { createAztecNodeClient } from '@aztec/circuit-types'; import { NULL_KEY } from '@aztec/ethereum'; import { ServerList, createMultiJsonRpcServer } from '@aztec/foundation/json-rpc/server'; import { DebugLogger, LogFn } from '@aztec/foundation/log'; +import { AztecLmdbStore } from '@aztec/kv-store'; import { BootstrapNode, P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; import { PXEService, PXEServiceConfig, createPXERpcServer, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; @@ -19,7 +20,6 @@ import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; import { MNEMONIC, createAztecNode, createAztecPXE, createSandbox, deployContractsToL1 } from '../sandbox.js'; import { github, splash } from '../splash.js'; import { cliTexts } from './texts.js'; -import { openDb } from './util.js'; const { AZTEC_PORT = '8080', AZTEC_NODE_URL, DEPLOY_AZTEC_CONTRACTS } = process.env; @@ -239,8 +239,11 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { // merge env vars and cli options const archiverConfig = pick({ ...archiverConfigEnvVars, ...archiverCliOptions }); - const nodeDb = await openDb(archiverConfig, debugLogger); - const archiverStore = new LMDBArchiverStore(nodeDb, archiverConfig.maxLogs); + const store = await AztecLmdbStore.create( + archiverConfig.l1Contracts.rollupAddress, + archiverConfig.dataDirectory, + ); + const archiverStore = new KVArchiverDataStore(store, archiverConfig.maxLogs); const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, true); const archiverServer = createArchiverRpcServer(archiver); diff --git a/yarn-project/aztec/src/cli/util.ts b/yarn-project/aztec/src/cli/util.ts deleted file mode 100644 index 33f68f78974..00000000000 --- a/yarn-project/aztec/src/cli/util.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { ArchiverConfig } from '@aztec/archiver'; -import { LogFn } from '@aztec/foundation/log'; - -import { LevelDown, default as leveldown } from 'leveldown'; -import { RootDatabase, open } from 'lmdb'; -import { MemDown, default as memdown } from 'memdown'; -import { mkdir } from 'node:fs/promises'; -import { join } from 'node:path'; - -export const createMemDown = () => (memdown as any)() as MemDown; -export const createLevelDown = (path: string) => (leveldown as any)(path) as LevelDown; - -const DB_SUBDIR = 'aztec-archiver-db'; -const ARCHIVER_METADATA_KEY = '@@aztec_archiver_metadata'; - -/** - * The metadata for an aztec node. - */ -type ArchiverMetadata = { - /** - * The address of the rollup contract on L1 - */ - rollupContractAddress: string; -}; - -/** - * Opens the database for the archiver. If a data directory is specified, then this attempts to create it. - * @param config - The configuration to be used by the archiver. - * @throws If `config.dataDirectory` is set and the directory cannot be created. - * @returns The database instance for the archiver. - */ -export async function openDb(config: ArchiverConfig, log: LogFn): Promise { - const archiverMetadata: ArchiverMetadata = { - rollupContractAddress: config.l1Contracts.rollupAddress.toString(), - }; - - let archiverDb: RootDatabase; - - if (config.dataDirectory) { - const dir = join(config.dataDirectory, DB_SUBDIR); - // this throws if we don't have permissions to create the directory - await mkdir(dir, { recursive: true }); - - log(`Opening archiver database at ${dir}`); - archiverDb = open(dir, {}); - } else { - log('Opening temporary databases'); - // not passing a path will use a temp file that gets deleted when the process exits - archiverDb = open({}); - } - - await checkArchiverMetadataAndClear(archiverDb, archiverMetadata, log); - return archiverDb; -} - -/** - * Checks the archiver metadata and clears the database if the rollup contract address has changed. - * @param archiverDb - The database for the aztec archiver. - * @param archiverMetadata - The metadata for the aztec archiver. - */ -async function checkArchiverMetadataAndClear( - archiverDb: RootDatabase, - archiverMetadata: ArchiverMetadata, - log: LogFn, -): Promise { - const metadataDB = archiverDb.openDB('metadata', {}); - try { - const existing = metadataDB.get(ARCHIVER_METADATA_KEY); - // if the rollup addresses are different, wipe the local database and start over - if (!existing || existing.rollupContractAddress !== archiverMetadata.rollupContractAddress) { - log('Rollup contract address has changed, clearing databases'); - await archiverDb.clearAsync(); - } - await metadataDB.put(ARCHIVER_METADATA_KEY, archiverMetadata); - } finally { - await metadataDB.close(); - } -} diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 97b2f9ec276..031f3360dd2 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -265,6 +265,7 @@ __metadata: "@aztec/circuits.js": "workspace:^" "@aztec/ethereum": "workspace:^" "@aztec/foundation": "workspace:^" + "@aztec/kv-store": "workspace:^" "@aztec/l1-artifacts": "workspace:^" "@aztec/noir-compiler": "workspace:^" "@aztec/noir-contracts": "workspace:^" From 4bfae4e6e369d84d063901b0fe8165e9d4b49517 Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 19 Jan 2024 15:05:45 +0000 Subject: [PATCH 38/50] yarn prepare --- yarn-project/aztec/tsconfig.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yarn-project/aztec/tsconfig.json b/yarn-project/aztec/tsconfig.json index 4844bd37232..a27fb82841d 100644 --- a/yarn-project/aztec/tsconfig.json +++ b/yarn-project/aztec/tsconfig.json @@ -30,6 +30,9 @@ { "path": "../foundation" }, + { + "path": "../kv-store" + }, { "path": "../l1-artifacts" }, From d5500079915669c4bcec9f7e51ccf4ae83fbdc6f Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 19 Jan 2024 15:11:04 +0000 Subject: [PATCH 39/50] update sandbox reference --- docs/docs/dev_docs/cli/sandbox-reference.md | 23 +++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/docs/dev_docs/cli/sandbox-reference.md b/docs/docs/dev_docs/cli/sandbox-reference.md index 97dfc98998a..8fb0866d331 100644 --- a/docs/docs/dev_docs/cli/sandbox-reference.md +++ b/docs/docs/dev_docs/cli/sandbox-reference.md @@ -56,13 +56,24 @@ cd ~/.aztec && docker-compose up ## Running Aztec PXE / Node / P2P-Bootstrap node -If you wish to run components of the Aztec network stack separately, you can still use the Sandbox by including a `MODE` variable. -The values for `MODE` can be: +If you wish to run components of the Aztec network stack separately, you can use the `aztec start` command with various options for enabling components. -- sandbox (default) -- node -- pxe -- p2p-bootstrap +```bash +aztec start --node [nodeOptions] --pxe [pxeOptions] --archiver [archiverOptions] --sequencer [sequencerOptions] ----p2p-bootstrap [p2pOptions] +``` + +Starting the aztec node alongside a PXE, sequencer or archiver, will attach the components to the node. If you want to e.g. run a PXE separately to a node, you can: +Start a node: + +```bash +aztec start --node [node] --archiver [archiverOptions] +``` + +Then start a PXE on a separate terminal that connects to that node: + +```bash +aztec start --pxe nodeUrl=http://localhost:8080 +``` ## Environment Variables From b9db631719adc8b07bddfe5c3df9412d8a6aa9ff Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 19 Jan 2024 18:15:36 +0000 Subject: [PATCH 40/50] Addressing PR comments --- yarn-project/aztec-node/package.json | 1 - yarn-project/aztec-node/tsconfig.json | 3 --- yarn-project/aztec/Dockerfile | 2 +- yarn-project/aztec/src/cli/cli.ts | 3 +-- yarn-project/deploy_npm.sh | 1 - yarn-project/yarn.lock | 1 - 6 files changed, 2 insertions(+), 9 deletions(-) diff --git a/yarn-project/aztec-node/package.json b/yarn-project/aztec-node/package.json index 50fc850ceba..52b8ccbe5fb 100644 --- a/yarn-project/aztec-node/package.json +++ b/yarn-project/aztec-node/package.json @@ -34,7 +34,6 @@ }, "dependencies": { "@aztec/archiver": "workspace:^", - "@aztec/aztec.js": "workspace:^", "@aztec/circuit-types": "workspace:^", "@aztec/circuits.js": "workspace:^", "@aztec/ethereum": "workspace:^", diff --git a/yarn-project/aztec-node/tsconfig.json b/yarn-project/aztec-node/tsconfig.json index 23f2a6a349f..9979b01f137 100644 --- a/yarn-project/aztec-node/tsconfig.json +++ b/yarn-project/aztec-node/tsconfig.json @@ -9,9 +9,6 @@ { "path": "../archiver" }, - { - "path": "../aztec.js" - }, { "path": "../circuit-types" }, diff --git a/yarn-project/aztec/Dockerfile b/yarn-project/aztec/Dockerfile index 2063bcc92ed..e8a8e8d45d1 100644 --- a/yarn-project/aztec/Dockerfile +++ b/yarn-project/aztec/Dockerfile @@ -1,6 +1,6 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/aztec/dest/bin/index.js"] -EXPOSE 79 80 +EXPOSE 8080 # The version has been updated in yarn-project-prod. # Adding COMMIT_TAG here to rebuild versioned image. diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index 19eedab36f3..f5f658ad4b8 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -13,7 +13,6 @@ import { PXEService, PXEServiceConfig, createPXERpcServer, createPXEService, get import { Command } from 'commander'; import { readFileSync } from 'fs'; import http from 'http'; -import pick from 'lodash.pick'; import { dirname, resolve } from 'path'; import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; @@ -237,7 +236,7 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { // get config from options const archiverCliOptions = parseModuleOptions(options.pxe); // merge env vars and cli options - const archiverConfig = pick({ ...archiverConfigEnvVars, ...archiverCliOptions }); + const archiverConfig = { ...archiverConfigEnvVars, ...archiverCliOptions }; const store = await AztecLmdbStore.create( archiverConfig.l1Contracts.rollupAddress, diff --git a/yarn-project/deploy_npm.sh b/yarn-project/deploy_npm.sh index 1588469a49f..9132db974d0 100755 --- a/yarn-project/deploy_npm.sh +++ b/yarn-project/deploy_npm.sh @@ -99,4 +99,3 @@ deploy_package p2p deploy_package world-state deploy_package sequencer-client deploy_package aztec-node -deploy_package aztec diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 031f3360dd2..0b9ea3518c8 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -173,7 +173,6 @@ __metadata: resolution: "@aztec/aztec-node@workspace:aztec-node" dependencies: "@aztec/archiver": "workspace:^" - "@aztec/aztec.js": "workspace:^" "@aztec/circuit-types": "workspace:^" "@aztec/circuits.js": "workspace:^" "@aztec/ethereum": "workspace:^" From 884b4029212c83800142e5ba3dac1e86af456b74 Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 19 Jan 2024 18:18:35 +0000 Subject: [PATCH 41/50] Remove lodash.pick from aztec package --- yarn-project/aztec/package.json | 1 - yarn-project/yarn.lock | 1 - 2 files changed, 2 deletions(-) diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index 98163679e41..f1b3cc41959 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -47,7 +47,6 @@ "koa-router": "^12.0.0", "leveldown": "^6.1.1", "lmdb": "^2.9.2", - "lodash.pick": "^4.4.0", "memdown": "^6.1.1", "viem": "^1.2.5", "winston": "^3.10.0", diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 0b9ea3518c8..a6d75d5b09e 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -280,7 +280,6 @@ __metadata: koa-router: ^12.0.0 leveldown: ^6.1.1 lmdb: ^2.9.2 - lodash.pick: ^4.4.0 memdown: ^6.1.1 ts-jest: ^29.1.0 ts-node: ^10.9.1 From 2c3605a5f0aee982c211c8681991e8981800dc94 Mon Sep 17 00:00:00 2001 From: spypsy Date: Mon, 22 Jan 2024 10:23:31 +0000 Subject: [PATCH 42/50] allow 0x private keys in aztec-cli --- yarn-project/cli/src/utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yarn-project/cli/src/utils.ts b/yarn-project/cli/src/utils.ts index a3926ae6750..ef2c07a811b 100644 --- a/yarn-project/cli/src/utils.ts +++ b/yarn-project/cli/src/utils.ts @@ -62,7 +62,9 @@ export async function deployAztecContracts( const { createEthereumChain, deployL1Contracts } = await import('@aztec/ethereum'); const { mnemonicToAccount, privateKeyToAccount } = await import('viem/accounts'); - const account = !privateKey ? mnemonicToAccount(mnemonic!) : privateKeyToAccount(`0x${privateKey}`); + const account = !privateKey + ? mnemonicToAccount(mnemonic!) + : privateKeyToAccount(`${privateKey.startsWith('0x') ? '' : '0x'}${privateKey}` as `0x${string}`); const chain = createEthereumChain(rpcUrl, apiKey); const l1Artifacts: L1ContractArtifactsForDeployment = { contractDeploymentEmitter: { From ed84f92a144148607bdd96e581e9c6b45f4badf9 Mon Sep 17 00:00:00 2001 From: spypsy Date: Mon, 22 Jan 2024 12:55:28 +0000 Subject: [PATCH 43/50] Fix archiver client + contract address requirements --- .../archiver/src/rpc/archiver_client.ts | 2 +- yarn-project/aztec/src/cli/cli.ts | 66 +++-------- yarn-project/aztec/src/cli/util.ts | 105 ++++++++++++++++++ yarn-project/cli/src/bin/index.ts | 2 +- 4 files changed, 121 insertions(+), 54 deletions(-) create mode 100644 yarn-project/aztec/src/cli/util.ts diff --git a/yarn-project/archiver/src/rpc/archiver_client.ts b/yarn-project/archiver/src/rpc/archiver_client.ts index 82fb795f214..89b9c4bb127 100644 --- a/yarn-project/archiver/src/rpc/archiver_client.ts +++ b/yarn-project/archiver/src/rpc/archiver_client.ts @@ -28,6 +28,6 @@ export const createArchiverClient = (url: string, fetch = makeFetch([1, 2, 3], t }, {}, false, - false, + 'archiver', fetch, ); diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index f5f658ad4b8..05c925cc893 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -1,5 +1,11 @@ import { deployInitialTestAccounts } from '@aztec/accounts/testing'; -import { Archiver, KVArchiverDataStore, createArchiverRpcServer, getConfigEnvVars } from '@aztec/archiver'; +import { + Archiver, + ArchiverConfig, + KVArchiverDataStore, + createArchiverRpcServer, + getConfigEnvVars, +} from '@aztec/archiver'; import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; import { AccountManager, GrumpkinScalar, fileURLToPath } from '@aztec/aztec.js'; import { createAztecNodeClient } from '@aztec/circuit-types'; @@ -19,58 +25,10 @@ import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; import { MNEMONIC, createAztecNode, createAztecPXE, createSandbox, deployContractsToL1 } from '../sandbox.js'; import { github, splash } from '../splash.js'; import { cliTexts } from './texts.js'; +import { installSignalHandlers, mergeEnvVarsAndCliOptions, parseModuleOptions } from './util.js'; const { AZTEC_PORT = '8080', AZTEC_NODE_URL, DEPLOY_AZTEC_CONTRACTS } = process.env; -const installSignalHandlers = (logFn: LogFn, cb?: Array<() => Promise>) => { - const shutdown = async () => { - logFn('Shutting down...'); - if (cb) { - await Promise.all(cb); - } - process.exit(0); - }; - process.removeAllListeners('SIGINT'); - process.removeAllListeners('SIGTERM'); - process.once('SIGINT', shutdown); - process.once('SIGTERM', shutdown); -}; - -/** - * Parses a string of options into a key-value map. - * @param options - String of options in the format "option1=value1,option2=value2". - * @returns Key-value map of options. - */ -const parseModuleOptions = (options: string): Record => { - if (!options?.length) { - return {}; - } - const optionsArray = options.split(','); - return optionsArray.reduce((acc, option) => { - const [key, value] = option.split('='); - return { ...acc, [key]: value }; - }, {}); -}; - -const mergeEnvVarsAndCliOptions = (envVars: Record, cliOptions: Record) => { - const cliOptionsContracts = { - rollupAddress: cliOptions.rollupAddress, - registryAddress: cliOptions.registryAddress, - inboxAddress: cliOptions.inboxAddress, - outboxAddress: cliOptions.outboxAddress, - contractDeploymentEmitterAddress: cliOptions.contractDeploymentEmitterAddress, - }; - const merged = { - ...envVars, - ...cliOptions, - l1Contracts: { - ...envVars.l1Contracts, - ...cliOptionsContracts, - }, - } as T; - return merged; -}; - /** * Returns commander program that defines the 'aztec' command line interface. * @param userLog - log function for logging user output. @@ -234,9 +192,13 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { // get env vars first const archiverConfigEnvVars = getConfigEnvVars(); // get config from options - const archiverCliOptions = parseModuleOptions(options.pxe); + const archiverCliOptions = parseModuleOptions(options.archiver); // merge env vars and cli options - const archiverConfig = { ...archiverConfigEnvVars, ...archiverCliOptions }; + const archiverConfig = mergeEnvVarsAndCliOptions( + archiverConfigEnvVars, + archiverCliOptions, + true, + ); const store = await AztecLmdbStore.create( archiverConfig.l1Contracts.rollupAddress, diff --git a/yarn-project/aztec/src/cli/util.ts b/yarn-project/aztec/src/cli/util.ts new file mode 100644 index 00000000000..061df95bcfc --- /dev/null +++ b/yarn-project/aztec/src/cli/util.ts @@ -0,0 +1,105 @@ +import { ArchiverConfig } from '@aztec/archiver'; +import { AztecNodeConfig } from '@aztec/aztec-node'; +import { L1ContractAddresses } from '@aztec/ethereum'; +import { EthAddress } from '@aztec/foundation/eth-address'; +import { LogFn } from '@aztec/foundation/log'; +import { P2PConfig } from '@aztec/p2p'; +import { PXEServiceConfig } from '@aztec/pxe'; + +/** + * Checks if the object has l1Contracts property + * @param obj - The object to check + * @returns True if the object has l1Contracts property + */ +function hasL1Contracts(obj: any): obj is { + /** the deployed L1 contract addresses */ + l1Contracts: unknown; +} { + return 'l1Contracts' in obj; +} + +/** + * Checks if all contract addresses set in config. + * @param contracts - L1 Contract Addresses object + * @returns true if all contract addresses are not zero + */ +const checkContractAddresses = (contracts: L1ContractAddresses) => { + return ['rollupAddress', 'inboxAddress', 'outboxAddress', 'contractDeploymentEmitterAddress'].every(cn => { + const key = cn as keyof L1ContractAddresses; + return contracts[key] && contracts[key] !== EthAddress.ZERO; + }); +}; + +export const installSignalHandlers = (logFn: LogFn, cb?: Array<() => Promise>) => { + const shutdown = async () => { + logFn('Shutting down...'); + if (cb) { + await Promise.all(cb); + } + process.exit(0); + }; + process.removeAllListeners('SIGINT'); + process.removeAllListeners('SIGTERM'); + process.once('SIGINT', shutdown); + process.once('SIGTERM', shutdown); +}; + +/** + * Parses a string of options into a key-value map. + * @param options - String of options in the format "option1=value1,option2=value2". + * @returns Key-value map of options. + */ +export const parseModuleOptions = (options: string): Record => { + if (!options?.length) { + return {}; + } + const optionsArray = options.split(','); + return optionsArray.reduce((acc, option) => { + const [key, value] = option.split('='); + return { ...acc, [key]: value }; + }, {}); +}; + +export const mergeEnvVarsAndCliOptions = ( + envVars: AztecNodeConfig | PXEServiceConfig | P2PConfig | ArchiverConfig, + cliOptions: Record, + contractsRequired = false, +) => { + if (contractsRequired && !cliOptions.rollupAddress) { + throw new Error('Rollup contract address is required to start the service'); + } + const cliOptionsContracts: L1ContractAddresses = { + rollupAddress: cliOptions.rollupAddress ? EthAddress.fromString(cliOptions.rollupAddress) : EthAddress.ZERO, + registryAddress: cliOptions.registryAddress ? EthAddress.fromString(cliOptions.registryAddress) : EthAddress.ZERO, + inboxAddress: cliOptions.inboxAddress ? EthAddress.fromString(cliOptions.inboxAddress) : EthAddress.ZERO, + outboxAddress: cliOptions.outboxAddress ? EthAddress.fromString(cliOptions.outboxAddress) : EthAddress.ZERO, + contractDeploymentEmitterAddress: cliOptions.contractDeploymentEmitterAddress + ? EthAddress.fromString(cliOptions.contractDeploymentEmitterAddress) + : EthAddress.ZERO, + availabilityOracleAddress: cliOptions.availabilityOracleAddress + ? EthAddress.fromString(cliOptions.availabilityOracleAddress) + : EthAddress.ZERO, + }; + + if ( + hasL1Contracts(envVars) && + contractsRequired && + (!checkContractAddresses(cliOptionsContracts) || !checkContractAddresses(envVars.l1Contracts)) + ) { + throw new Error('Deployed L1 contract addresses are required to start the service'); + } + + let merged = { ...envVars, ...cliOptions } as T; + + if (hasL1Contracts(envVars)) { + merged = { + ...merged, + l1Contracts: { + ...(envVars.l1Contracts && { ...envVars.l1Contracts }), + ...cliOptionsContracts, + }, + } as T; + } + + return merged; +}; diff --git a/yarn-project/cli/src/bin/index.ts b/yarn-project/cli/src/bin/index.ts index 4e1c4b1ad28..0d79e796d50 100644 --- a/yarn-project/cli/src/bin/index.ts +++ b/yarn-project/cli/src/bin/index.ts @@ -5,7 +5,7 @@ import 'source-map-support/register.js'; import { getProgram } from '../index.js'; -const debugLogger = createDebugLogger('aztec:cli'); +const debugLogger = createDebugLogger('aztec:cli-client'); const log = createConsoleLogger(); /** CLI main entrypoint */ From 45b41b6bf550da9499378de546e2d18e37d2df8e Mon Sep 17 00:00:00 2001 From: spypsy Date: Mon, 22 Jan 2024 13:33:08 +0000 Subject: [PATCH 44/50] namespace rpc server by default, remove unused 'useApi' option --- .../archiver/src/rpc/archiver_server.ts | 1 - .../src/aztec-node/http_rpc_server.ts | 1 - yarn-project/aztec/src/cli/cli.ts | 6 +- yarn-project/end-to-end/src/e2e_cli.test.ts | 2 +- .../json-rpc/client/json_rpc_client.test.ts | 24 +++- .../json-rpc/server/json_rpc_server.test.ts | 15 +-- .../src/json-rpc/server/json_rpc_server.ts | 117 ++++++------------ .../pxe/src/pxe_http/pxe_http_server.ts | 1 - 8 files changed, 70 insertions(+), 97 deletions(-) diff --git a/yarn-project/archiver/src/rpc/archiver_server.ts b/yarn-project/archiver/src/rpc/archiver_server.ts index 77cb761ae0b..7c011e5d242 100644 --- a/yarn-project/archiver/src/rpc/archiver_server.ts +++ b/yarn-project/archiver/src/rpc/archiver_server.ts @@ -32,7 +32,6 @@ export function createArchiverRpcServer(archiverService: Archiver): JsonRpcServe L2BlockL2Logs, }, {}, - false, ['start', 'stop'], ); } diff --git a/yarn-project/aztec-node/src/aztec-node/http_rpc_server.ts b/yarn-project/aztec-node/src/aztec-node/http_rpc_server.ts index f5e99becbde..e7a3f80a802 100644 --- a/yarn-project/aztec-node/src/aztec-node/http_rpc_server.ts +++ b/yarn-project/aztec-node/src/aztec-node/http_rpc_server.ts @@ -43,7 +43,6 @@ export function createAztecNodeRpcServer(node: AztecNode) { L1ToL2MessageAndIndex, }, { Tx, L2BlockL2Logs }, - false, // disable methods not part of the AztecNode interface ['start', 'stop'], ); diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index 05c925cc893..16d14cdc814 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -10,7 +10,7 @@ import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeC import { AccountManager, GrumpkinScalar, fileURLToPath } from '@aztec/aztec.js'; import { createAztecNodeClient } from '@aztec/circuit-types'; import { NULL_KEY } from '@aztec/ethereum'; -import { ServerList, createMultiJsonRpcServer } from '@aztec/foundation/json-rpc/server'; +import { ServerList, createNamespacedJsonRpcServer } from '@aztec/foundation/json-rpc/server'; import { DebugLogger, LogFn } from '@aztec/foundation/log'; import { AztecLmdbStore } from '@aztec/kv-store'; import { BootstrapNode, P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; @@ -69,7 +69,7 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { // Start Node and PXE JSON-RPC server const nodeServer = createAztecNodeRpcServer(node); const pxeServer = createPXERpcServer(pxe); - const rpcServer = createMultiJsonRpcServer([{ node: nodeServer }, { pxe: pxeServer }], debugLogger); + const rpcServer = createNamespacedJsonRpcServer([{ node: nodeServer }, { pxe: pxeServer }], debugLogger); const app = rpcServer.getApp(); const httpServer = http.createServer(app.callback()); @@ -220,7 +220,7 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { signalHandlers.push(bootstrapNode.stop); } if (services.length) { - const rpcServer = createMultiJsonRpcServer(services, debugLogger); + const rpcServer = createNamespacedJsonRpcServer(services, debugLogger); const app = rpcServer.getApp(); const httpServer = http.createServer(app.callback()); diff --git a/yarn-project/end-to-end/src/e2e_cli.test.ts b/yarn-project/end-to-end/src/e2e_cli.test.ts index 221902983f9..7fd0884a091 100644 --- a/yarn-project/end-to-end/src/e2e_cli.test.ts +++ b/yarn-project/end-to-end/src/e2e_cli.test.ts @@ -20,7 +20,7 @@ const testSetup = async () => { debug(`Environment set up`); ({ pxe, teardown } = context); if (!RPC_URL) { - http = startHttpRpcServer(pxe, createPXERpcServer, HTTP_PORT); + http = startHttpRpcServer('pxe', pxe, createPXERpcServer, HTTP_PORT); debug(`HTTP RPC server started on port ${HTTP_PORT}`); RPC_URL = `http://localhost:${HTTP_PORT}`; } diff --git a/yarn-project/foundation/src/json-rpc/client/json_rpc_client.test.ts b/yarn-project/foundation/src/json-rpc/client/json_rpc_client.test.ts index 2e2c61aa02b..c95984230a2 100644 --- a/yarn-project/foundation/src/json-rpc/client/json_rpc_client.test.ts +++ b/yarn-project/foundation/src/json-rpc/client/json_rpc_client.test.ts @@ -1,13 +1,13 @@ import request from 'supertest'; import { TestNote, TestState } from '../fixtures/test_state.js'; -import { JsonRpcServer } from '../server/index.js'; +import { JsonRpcServer, createNamespacedJsonRpcServer } from '../server/index.js'; import { createJsonRpcClient } from './json_rpc_client.js'; it('test an RPC function over client', async () => { const mockFetch = async (host: string, method: string, body: any) => { - const server = new JsonRpcServer(new TestState([new TestNote('a'), new TestNote('b')]), { TestNote }, {}, true); - const result = await request(server.getApp().callback()).post(`/${method}`).send(body); + const server = new JsonRpcServer(new TestState([new TestNote('a'), new TestNote('b')]), { TestNote }, {}); + const result = await request(server.getApp().callback()).post(`/`).send(body); return JSON.parse(result.text); }; const client = createJsonRpcClient('', { TestNote }, {}, true, false, mockFetch); @@ -19,3 +19,21 @@ it('test an RPC function over client', async () => { expect(result[1].toString()).toBe('b'); expect(result[2].toString()).toBe('c'); }); + +it('test a namespaced RPC function over client', async () => { + const namespace = 'testService'; + const mockFetch = async (host: string, method: string, body: any) => { + const service = new JsonRpcServer(new TestState([new TestNote('a'), new TestNote('b')]), { TestNote }, {}); + const server = createNamespacedJsonRpcServer([{ [namespace]: service }]); + const result = await request(server.getApp().callback()).post('/').send(body); + return JSON.parse(result.text); + }; + const client = createJsonRpcClient('', { TestNote }, {}, true, namespace, mockFetch); + const result = await client.addNotes([new TestNote('c')]); + expect(result[0]).toBeInstanceOf(TestNote); + expect(result[1]).toBeInstanceOf(TestNote); + expect(result[2]).toBeInstanceOf(TestNote); + expect(result[0].toString()).toBe('a'); + expect(result[1].toString()).toBe('b'); + expect(result[2].toString()).toBe('c'); +}); diff --git a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.test.ts b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.test.ts index c5272c2b74c..9815055346d 100644 --- a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.test.ts +++ b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.test.ts @@ -4,19 +4,20 @@ import { TestNote, TestState } from '../fixtures/test_state.js'; import { JsonRpcServer } from './json_rpc_server.js'; it('test an RPC function with a primitive parameter', async () => { - const server = new JsonRpcServer(new TestState([new TestNote('a'), new TestNote('b')]), { TestNote }, {}, true); + const server = new JsonRpcServer(new TestState([new TestNote('a'), new TestNote('b')]), { TestNote }, {}); const response = await request(server.getApp().callback()) - .post('/getNote') - .send({ params: [0] }); + .post('/') + .send({ method: 'getNote', params: [0] }); expect(response.status).toBe(200); expect(response.text).toBe(JSON.stringify({ result: { type: 'TestNote', data: 'a' } })); }); it('test an RPC function with an array of classes', async () => { - const server = new JsonRpcServer(new TestState([]), { TestNote }, {}, true); + const server = new JsonRpcServer(new TestState([]), { TestNote }, {}); const response = await request(server.getApp().callback()) - .post('/addNotes') + .post('/') .send({ + method: 'addNotes', params: [[{ data: 'a' }, { data: 'b' }, { data: 'c' }]], }); expect(response.status).toBe(200); @@ -24,7 +25,7 @@ it('test an RPC function with an array of classes', async () => { }); it('test invalid JSON', async () => { - const server = new JsonRpcServer(new TestState([]), { TestNote }, {}, false); + const server = new JsonRpcServer(new TestState([]), { TestNote }, {}); const response = await request(server.getApp().callback()).post('/').send('{'); expect(response.status).toBe(400); expect(response.body).toEqual({ @@ -35,7 +36,7 @@ it('test invalid JSON', async () => { }); it('invalid method', async () => { - const server = new JsonRpcServer(new TestState([]), { TestNote }, {}, false); + const server = new JsonRpcServer(new TestState([]), { TestNote }, {}); const response = await request(server.getApp().callback()).post('/').send({ jsonrpc: '2.0', method: 'invalid', diff --git a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts index c2d6ff6f77a..08e0533aff4 100644 --- a/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts +++ b/yarn-project/foundation/src/json-rpc/server/json_rpc_server.ts @@ -23,7 +23,6 @@ export class JsonRpcServer { private handler: object, private stringClassMap: StringClassConverterInput, private objectClassMap: JsonClassConverterInput, - private createApi: boolean, /** List of methods to disallow from calling remotely */ public readonly disallowedMethods: string[] = [], private log = createDebugLogger('aztec:foundation:json-rpc:server'), @@ -94,90 +93,45 @@ export class JsonRpcServer { private getRouter(prefix: string) { const router = new Router({ prefix }); const proto = Object.getPrototypeOf(this.handler); - // Find all our endpoints from the handler methods - - if (this.createApi) { - // "API mode" where an endpoint is created for each method - for (const method of Object.getOwnPropertyNames(proto)) { - // Ignore if not a function or function is not allowed - if ( - method === 'constructor' || - typeof proto[method] !== 'function' || - this.disallowedMethods.includes(method) - ) { - continue; - } - router.post(`/${method}`, async (ctx: Koa.Context) => { - const { params = [], jsonrpc, id } = ctx.request.body as any; - try { - const result = await this.proxy.call(method, params); - ctx.body = { - jsonrpc, - id, - result: convertBigintsInObj(result), - }; - ctx.status = 200; - } catch (err: any) { - // Propagate the error message to the client. Plenty of the errors are expected to occur (e.g. adding - // a duplicate recipient) so this is necessary. - ctx.status = 400; - ctx.body = { - jsonrpc, - id, - error: { - // TODO assign error codes - https://github.com/AztecProtocol/aztec-packages/issues/2633 - code: -32000, - message: err.message, - }, - }; - } - }); - } - } else { - // "JSON RPC mode" where a single endpoint is used and the method is given in the request body - router.post('/', async (ctx: Koa.Context) => { - const { params = [], jsonrpc, id, method } = ctx.request.body as any; - // Ignore if not a function - if ( - method === 'constructor' || - typeof proto[method] !== 'function' || - this.disallowedMethods.includes(method) - ) { + // "JSON RPC mode" where a single endpoint is used and the method is given in the request body + router.post('/', async (ctx: Koa.Context) => { + const { params = [], jsonrpc, id, method } = ctx.request.body as any; + // Ignore if not a function + if (method === 'constructor' || typeof proto[method] !== 'function' || this.disallowedMethods.includes(method)) { + ctx.status = 400; + ctx.body = { + jsonrpc, + id, + error: { + code: -32601, + message: `Method not found: ${method}`, + }, + }; + } else { + try { + const result = await this.proxy.call(method, params); + ctx.body = { + jsonrpc, + id, + result: convertBigintsInObj(result), + }; + ctx.status = 200; + } catch (err: any) { + // Propagate the error message to the client. Plenty of the errors are expected to occur (e.g. adding + // a duplicate recipient) so this is necessary. ctx.status = 400; ctx.body = { jsonrpc, id, error: { - code: -32601, - message: `Method not found: ${method}`, + // TODO assign error codes - https://github.com/AztecProtocol/aztec-packages/issues/2633 + code: -32000, + message: err.message, }, }; - } else { - try { - const result = await this.proxy.call(method, params); - ctx.body = { - jsonrpc, - id, - result: convertBigintsInObj(result), - }; - ctx.status = 200; - } catch (err: any) { - // Propagate the error message to the client. Plenty of the errors are expected to occur (e.g. adding - // a duplicate recipient) so this is necessary. - ctx.status = 400; - ctx.body = { - jsonrpc, - id, - error: { - // TODO assign error codes - https://github.com/AztecProtocol/aztec-packages/issues/2633 - code: -32000, - message: err.message, - }, - }; - } } - }); - } + } + }); return router; } @@ -241,13 +195,16 @@ export function createStatusRouter(apiPrefix = '') { * @returns A running http server. */ export function startHttpRpcServer( + name: string, instance: T, jsonRpcFactoryFunc: (instance: T) => JsonRpcServer, port: string | number, ): http.Server { const rpcServer = jsonRpcFactoryFunc(instance); - const app = rpcServer.getApp(); + const namespacedServer = createNamespacedJsonRpcServer([{ [name]: rpcServer }]); + + const app = namespacedServer.getApp(); const httpServer = http.createServer(app.callback()); httpServer.listen(port); @@ -267,7 +224,7 @@ export type ServerList = { * @param servers - List of servers to be combined into a single server, passed as ServerList. * @returns A single JsonRpcServer with namespaced methods. */ -export function createMultiJsonRpcServer( +export function createNamespacedJsonRpcServer( servers: ServerList, log = createDebugLogger('aztec:foundation:json-rpc:multi-server'), ): JsonRpcServer { @@ -308,5 +265,5 @@ export function createMultiJsonRpcServer( { stringClassMap: {}, objectClassMap: {} } as ClassMaps, ); - return new JsonRpcServer(Object.create(handler), classMaps.stringClassMap, classMaps.objectClassMap, false, [], log); + return new JsonRpcServer(Object.create(handler), classMaps.stringClassMap, classMaps.objectClassMap, [], log); } diff --git a/yarn-project/pxe/src/pxe_http/pxe_http_server.ts b/yarn-project/pxe/src/pxe_http/pxe_http_server.ts index 7a324933936..300da326589 100644 --- a/yarn-project/pxe/src/pxe_http/pxe_http_server.ts +++ b/yarn-project/pxe/src/pxe_http/pxe_http_server.ts @@ -52,7 +52,6 @@ export function createPXERpcServer(pxeService: PXE): JsonRpcServer { LogId, }, { Tx, TxReceipt, L2BlockL2Logs }, - false, ['start', 'stop'], ); } From af7d63ddfd6fc1efa95fe04898c3656b3e661cf8 Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 23 Jan 2024 17:48:41 +0000 Subject: [PATCH 45/50] 'aztec sandbox' starts the full sandbox --- aztec-up/bin/aztec | 19 ++++++-- aztec-up/bin/aztec-cli | 3 +- aztec-up/bin/docker-compose.yml | 1 - yarn-project/aztec/src/bin/index.ts | 54 ++++++++++++++++++-- yarn-project/aztec/src/cli/cli.ts | 76 ++--------------------------- yarn-project/aztec/src/cli/util.ts | 36 +++++++++++++- 6 files changed, 106 insertions(+), 83 deletions(-) diff --git a/aztec-up/bin/aztec b/aztec-up/bin/aztec index 16402c7be85..93c53907c14 100755 --- a/aztec-up/bin/aztec +++ b/aztec-up/bin/aztec @@ -2,9 +2,20 @@ set -euo pipefail # Call cli image if used with `aztec cli ...args` -if [ "$1" == "cli" ]; then - shift - $(dirname $0)/.aztec-run aztecprotocol/cli $@ +if [ -n "${1-}" ] && [ "$1" != "--help" ]; then + if [ "$1" == "cli" ]; then + shift + $(dirname $0)/.aztec-run aztecprotocol/cli "$@" + elif [ "$1" == "sandbox" ]; then + $(dirname $0)/aztec-sandbox + else + $(dirname $0)/.aztec-run aztecprotocol/aztec "$@" + fi else - $(dirname $0)/.aztec-run aztecprotocol/aztec $@ + # TODO - display help message + echo + echo "Using 'aztec' CLI:" + echo " aztec start - Start aztec infrastructure components. See 'aztec start --help' for detailed command info." + echo " aztec sandbox - Run a local sandbox network (same as aztec-sandbox)." + echo " aztec cli - Run the aztec client CLI. See 'aztec cli --help' for detailed command info." fi diff --git a/aztec-up/bin/aztec-cli b/aztec-up/bin/aztec-cli index 6bdbb0473ba..3624338b5c3 100755 --- a/aztec-up/bin/aztec-cli +++ b/aztec-up/bin/aztec-cli @@ -3,5 +3,6 @@ set -euo pipefail export ENV_VARS_TO_INJECT="PXE_URL PRIVATE_KEY DEBUG" export PXE_URL=${PXE_URL:-"http://host.docker.internal:8080"} +export ETHEREUM_HOST=${ETHEREUM_HOST:-"http://host.docker.internal:8545"} -$(dirname $0)/.aztec-run aztecprotocol/cli $@ \ No newline at end of file +$(dirname $0)/.aztec-run aztecprotocol/cli $@ diff --git a/aztec-up/bin/docker-compose.yml b/aztec-up/bin/docker-compose.yml index 425a9be9636..7b3cab5e311 100644 --- a/aztec-up/bin/docker-compose.yml +++ b/aztec-up/bin/docker-compose.yml @@ -18,7 +18,6 @@ services: aztec: image: "aztecprotocol/aztec" - command: sandbox ports: - "${PXE_PORT:-8080}:${PXE_PORT:-8080}" environment: diff --git a/yarn-project/aztec/src/bin/index.ts b/yarn-project/aztec/src/bin/index.ts index 9da1de41f17..b463e48c895 100644 --- a/yarn-project/aztec/src/bin/index.ts +++ b/yarn-project/aztec/src/bin/index.ts @@ -1,14 +1,62 @@ +import { deployInitialTestAccounts } from '@aztec/accounts/testing'; +import { createAztecNodeRpcServer } from '@aztec/aztec-node'; +import { fileURLToPath } from '@aztec/aztec.js'; +import { createNamespacedJsonRpcServer } from '@aztec/foundation/json-rpc/server'; import { createConsoleLogger, createDebugLogger } from '@aztec/foundation/log'; +import { createPXERpcServer } from '@aztec/pxe'; + +import { readFileSync } from 'fs'; +import http from 'http'; +import { dirname, resolve } from 'path'; import { getProgram } from '../cli/index.js'; +import { createAccountLogs, installSignalHandlers } from '../cli/util.js'; +import { createSandbox } from '../sandbox.js'; +import { github, splash } from '../splash.js'; const userLog = createConsoleLogger(); const debugLogger = createDebugLogger('aztec:cli'); -/** CLI main entrypoint */ +const packageJsonPath = resolve(dirname(fileURLToPath(import.meta.url)), '../../package.json'); +const cliVersion: string = JSON.parse(readFileSync(packageJsonPath).toString()).version; + +const { TEST_ACCOUNTS = 'true', PORT = '8080' } = process.env; + +/** CLI & full node main entrypoint */ async function main() { - const program = getProgram(userLog, debugLogger); - await program.parseAsync(process.argv); + if (process.argv.length > 2) { + // If CLI arguments were provided, run the CLI program. + const cliProgram = getProgram(userLog, debugLogger); + await cliProgram.parseAsync(process.argv); + } else { + // If no CLI arguments were provided, run aztec full node for sandbox usage. + userLog(`${splash}\n${github}\n\n`); + userLog(`Setting up Aztec Sandbox v${cliVersion}, please stand by...`); + const { aztecNodeConfig, node, pxe, stop } = await createSandbox(); + installSignalHandlers(userLog, [stop]); + + // Deploy test accounts by default + if (TEST_ACCOUNTS === 'true') { + if (aztecNodeConfig.p2pEnabled) { + userLog(`Not setting up test accounts as we are connecting to a network`); + } else { + userLog('Setting up test accounts...'); + const accounts = await deployInitialTestAccounts(pxe); + const accLogs = await createAccountLogs(accounts, pxe); + userLog(accLogs.join('')); + } + } + + // Start Node and PXE JSON-RPC server + const nodeServer = createAztecNodeRpcServer(node); + const pxeServer = createPXERpcServer(pxe); + const rpcServer = createNamespacedJsonRpcServer([{ node: nodeServer }, { pxe: pxeServer }], debugLogger); + + const app = rpcServer.getApp(); + const httpServer = http.createServer(app.callback()); + httpServer.listen(PORT); + userLog(`Aztec Server listening on port ${PORT}`); + } } main().catch(err => { diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index 16d14cdc814..4c1cbaa7ab1 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -1,4 +1,3 @@ -import { deployInitialTestAccounts } from '@aztec/accounts/testing'; import { Archiver, ArchiverConfig, @@ -7,14 +6,14 @@ import { getConfigEnvVars, } from '@aztec/archiver'; import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; -import { AccountManager, GrumpkinScalar, fileURLToPath } from '@aztec/aztec.js'; +import { fileURLToPath } from '@aztec/aztec.js'; import { createAztecNodeClient } from '@aztec/circuit-types'; import { NULL_KEY } from '@aztec/ethereum'; import { ServerList, createNamespacedJsonRpcServer } from '@aztec/foundation/json-rpc/server'; import { DebugLogger, LogFn } from '@aztec/foundation/log'; import { AztecLmdbStore } from '@aztec/kv-store'; import { BootstrapNode, P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; -import { PXEService, PXEServiceConfig, createPXERpcServer, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; +import { PXEServiceConfig, createPXERpcServer, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import { Command } from 'commander'; import { readFileSync } from 'fs'; @@ -22,8 +21,7 @@ import http from 'http'; import { dirname, resolve } from 'path'; import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; -import { MNEMONIC, createAztecNode, createAztecPXE, createSandbox, deployContractsToL1 } from '../sandbox.js'; -import { github, splash } from '../splash.js'; +import { MNEMONIC, createAztecNode, createAztecPXE, deployContractsToL1 } from '../sandbox.js'; import { cliTexts } from './texts.js'; import { installSignalHandlers, mergeEnvVarsAndCliOptions, parseModuleOptions } from './util.js'; @@ -42,41 +40,6 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { program.name('aztec').description('Aztec command line interface').version(cliVersion); - // Start complete Sandbox. - program - .command('sandbox') - .description('Starts Aztec sandbox.') - .option('-p, --port ', 'Port to run Aztec on.', AZTEC_PORT) - .option('-s, --skip-test-accounts', 'DO NOT deploy test accounts.', false) - .action(async options => { - userLog(`${splash}\n${github}\n\n`); - userLog(`Setting up Aztec Sandbox v${cliVersion}, please stand by...`); - const { aztecNodeConfig, node, pxe, stop } = await createSandbox(); - installSignalHandlers(stop); - - // Deploy test accounts by default - if (!options.skipTestAccounts) { - if (aztecNodeConfig.p2pEnabled) { - userLog(`Not setting up test accounts as we are connecting to a network`); - } else { - userLog('Setting up test accounts...'); - const accounts = await deployInitialTestAccounts(pxe); - const accLogs = await createAccountLogs(accounts, pxe); - userLog(accLogs.join('')); - } - } - - // Start Node and PXE JSON-RPC server - const nodeServer = createAztecNodeRpcServer(node); - const pxeServer = createPXERpcServer(pxe); - const rpcServer = createNamespacedJsonRpcServer([{ node: nodeServer }, { pxe: pxeServer }], debugLogger); - - const app = rpcServer.getApp(); - const httpServer = http.createServer(app.callback()); - httpServer.listen(options.port); - userLog(`Aztec Server listening on port ${options.port}`); - }); - // Start Aztec modules with options program .command('start') @@ -231,36 +194,3 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { }); return program; } - -/** - * Creates logs for the initial accounts - * @param accounts - The initial accounts - * @param pxe - A PXE instance to get the registered accounts - * @returns A string array containing the initial accounts details - */ -async function createAccountLogs( - accounts: { - /** - * The account object - */ - account: AccountManager; - /** - * The private key of the account - */ - privateKey: GrumpkinScalar; - }[], - pxe: PXEService, -) { - const registeredAccounts = await pxe.getRegisteredAccounts(); - const accountLogStrings = [`Initial Accounts:\n\n`]; - for (const account of accounts) { - const completeAddress = account.account.getCompleteAddress(); - if (registeredAccounts.find(a => a.equals(completeAddress))) { - accountLogStrings.push(` Address: ${completeAddress.address.toString()}\n`); - accountLogStrings.push(` Partial Address: ${completeAddress.partialAddress.toString()}\n`); - accountLogStrings.push(` Private Key: ${account.privateKey.toString()}\n`); - accountLogStrings.push(` Public Key: ${completeAddress.publicKey.toString()}\n\n`); - } - } - return accountLogStrings; -} diff --git a/yarn-project/aztec/src/cli/util.ts b/yarn-project/aztec/src/cli/util.ts index 061df95bcfc..75f8c84d237 100644 --- a/yarn-project/aztec/src/cli/util.ts +++ b/yarn-project/aztec/src/cli/util.ts @@ -1,10 +1,11 @@ import { ArchiverConfig } from '@aztec/archiver'; import { AztecNodeConfig } from '@aztec/aztec-node'; +import { AccountManager } from '@aztec/aztec.js'; import { L1ContractAddresses } from '@aztec/ethereum'; import { EthAddress } from '@aztec/foundation/eth-address'; import { LogFn } from '@aztec/foundation/log'; import { P2PConfig } from '@aztec/p2p'; -import { PXEServiceConfig } from '@aztec/pxe'; +import { GrumpkinScalar, PXEService, PXEServiceConfig } from '@aztec/pxe'; /** * Checks if the object has l1Contracts property @@ -103,3 +104,36 @@ export const mergeEnvVarsAndCliOptions = a.equals(completeAddress))) { + accountLogStrings.push(` Address: ${completeAddress.address.toString()}\n`); + accountLogStrings.push(` Partial Address: ${completeAddress.partialAddress.toString()}\n`); + accountLogStrings.push(` Private Key: ${account.privateKey.toString()}\n`); + accountLogStrings.push(` Public Key: ${completeAddress.publicKey.toString()}\n\n`); + } + } + return accountLogStrings; +} From 374664d9afafb1fe6500576562c07d53e19071ab Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 24 Jan 2024 09:32:01 +0000 Subject: [PATCH 46/50] remove 'sandbox' command --- boxes/docker-compose.yml | 1 - yarn-project/aztec-node/terraform/main.tf | 1 - yarn-project/end-to-end/scripts/docker-compose-browser.yml | 1 - yarn-project/end-to-end/scripts/docker-compose.yml | 1 - 4 files changed, 4 deletions(-) diff --git a/boxes/docker-compose.yml b/boxes/docker-compose.yml index 3aa48f09860..e731d25c01e 100644 --- a/boxes/docker-compose.yml +++ b/boxes/docker-compose.yml @@ -6,7 +6,6 @@ services: aztec: image: aztecprotocol/aztec - command: sandbox environment: ETHEREUM_HOST: http://ethereum:8545 CHAIN_ID: 31337 diff --git a/yarn-project/aztec-node/terraform/main.tf b/yarn-project/aztec-node/terraform/main.tf index 6b505ab1ee9..a65230f6a16 100644 --- a/yarn-project/aztec-node/terraform/main.tf +++ b/yarn-project/aztec-node/terraform/main.tf @@ -155,7 +155,6 @@ resource "aws_ecs_task_definition" "aztec-node" { { "name": "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}", "image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}", - "command": ["sandbox"], "essential": true, "memoryReservation": 3776, "portMappings": [ diff --git a/yarn-project/end-to-end/scripts/docker-compose-browser.yml b/yarn-project/end-to-end/scripts/docker-compose-browser.yml index c812613848b..f859de7de39 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-browser.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-browser.yml @@ -14,7 +14,6 @@ services: sandbox: image: aztecprotocol/aztec:latest - command: sandbox environment: DEBUG: 'aztec:*' DEBUG_COLORS: 1 diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 8758c8a306c..6a05e652777 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -14,7 +14,6 @@ services: sandbox: image: aztecprotocol/aztec:latest - command: sandbox environment: DEBUG: 'aztec:*' DEBUG_COLORS: 1 From eb28fdc313353fe7436365096edc28572133ebaa Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 25 Jan 2024 14:33:46 +0000 Subject: [PATCH 47/50] PR Fixes --- yarn-project/aztec/src/cli/cli.ts | 148 ++---------------- .../aztec/src/cli/cmds/start_archiver.ts | 31 ++++ yarn-project/aztec/src/cli/cmds/start_node.ts | 87 ++++++++++ .../aztec/src/cli/cmds/start_p2p_bootstrap.ts | 18 +++ yarn-project/aztec/src/cli/cmds/start_pxe.ts | 38 +++++ yarn-project/aztec/src/cli/texts.ts | 9 +- 6 files changed, 190 insertions(+), 141 deletions(-) create mode 100644 yarn-project/aztec/src/cli/cmds/start_archiver.ts create mode 100644 yarn-project/aztec/src/cli/cmds/start_node.ts create mode 100644 yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts create mode 100644 yarn-project/aztec/src/cli/cmds/start_pxe.ts diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index 4c1cbaa7ab1..da6b97a454b 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -1,31 +1,16 @@ -import { - Archiver, - ArchiverConfig, - KVArchiverDataStore, - createArchiverRpcServer, - getConfigEnvVars, -} from '@aztec/archiver'; -import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; import { fileURLToPath } from '@aztec/aztec.js'; -import { createAztecNodeClient } from '@aztec/circuit-types'; -import { NULL_KEY } from '@aztec/ethereum'; import { ServerList, createNamespacedJsonRpcServer } from '@aztec/foundation/json-rpc/server'; import { DebugLogger, LogFn } from '@aztec/foundation/log'; -import { AztecLmdbStore } from '@aztec/kv-store'; -import { BootstrapNode, P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; -import { PXEServiceConfig, createPXERpcServer, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import { Command } from 'commander'; import { readFileSync } from 'fs'; import http from 'http'; import { dirname, resolve } from 'path'; -import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; -import { MNEMONIC, createAztecNode, createAztecPXE, deployContractsToL1 } from '../sandbox.js'; import { cliTexts } from './texts.js'; -import { installSignalHandlers, mergeEnvVarsAndCliOptions, parseModuleOptions } from './util.js'; +import { installSignalHandlers } from './util.js'; -const { AZTEC_PORT = '8080', AZTEC_NODE_URL, DEPLOY_AZTEC_CONTRACTS } = process.env; +const { AZTEC_PORT = '8080' } = process.env; /** * Returns commander program that defines the 'aztec' command line interface. @@ -53,134 +38,23 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { .option('-s, --sequencer [options]', cliTexts.sequencer) .option('-p2p, --p2p-bootstrap [options]', cliTexts.p2pBootstrap) .action(async options => { - // Services that will be started in a single multi-rpc server - const services: ServerList = []; // list of 'stop' functions to call when process ends const signalHandlers: Array<() => Promise> = []; + let services: ServerList = []; // Start Aztec Node if (options.node) { - // get env vars first - const aztecNodeConfigEnvVars = getNodeConfigEnvVars(); - // get config from options - const nodeCliOptions = parseModuleOptions(options.node); - // merge env vars and cli options - let nodeConfig = mergeEnvVarsAndCliOptions(aztecNodeConfigEnvVars, nodeCliOptions); - - // if no publisher private key, then use MNEMONIC - if (!options.archiver) { - // expect archiver url in node config - const archiverUrl = nodeCliOptions.archiverUrl; - if (!archiverUrl) { - userLog('Archiver Service URL is required to start Aztec Node without --archiver option'); - throw new Error('Archiver Service URL is required to start Aztec Node without --archiver option'); - } - nodeConfig.archiverUrl = archiverUrl; - } else { - const archiverCliOptions = parseModuleOptions(options.archiver); - nodeConfig = mergeEnvVarsAndCliOptions(aztecNodeConfigEnvVars, archiverCliOptions); - } - - // Deploy contracts if needed - if (nodeCliOptions.deployAztecContracts || DEPLOY_AZTEC_CONTRACTS === 'true') { - let account; - if (nodeConfig.publisherPrivateKey === NULL_KEY) { - account = mnemonicToAccount(MNEMONIC); - } else { - account = privateKeyToAccount(nodeConfig.publisherPrivateKey); - } - await deployContractsToL1(nodeConfig, account); - } - - if (!options.sequencer) { - nodeConfig.disableSequencer = true; - } else if (nodeConfig.publisherPrivateKey === NULL_KEY) { - // If we have a sequencer, ensure there's a publisher private key set. - const hdAccount = mnemonicToAccount(MNEMONIC); - const privKey = hdAccount.getHdKey().privateKey; - nodeConfig.publisherPrivateKey = `0x${Buffer.from(privKey!).toString('hex')}`; - } - - // Create and start Aztec Node. - const node = await createAztecNode(nodeConfig); - const nodeServer = createAztecNodeRpcServer(node); - - // Add node to services list - services.push({ node: nodeServer }); - - // Add node stop function to signal handlers - signalHandlers.push(node.stop); - - // Create a PXE client that connects to the node. - if (options.pxe) { - const pxeCliOptions = parseModuleOptions(options.pxe); - const pxeConfig = mergeEnvVarsAndCliOptions(getPXEServiceConfig(), pxeCliOptions); - const pxe = await createAztecPXE(node, pxeConfig); - const pxeServer = createPXERpcServer(pxe); - - // Add PXE to services list - services.push({ pxe: pxeServer }); - - // Add PXE stop function to signal handlers - signalHandlers.push(pxe.stop); - } + const { startNode } = await import('./cmds/start_node.js'); + services = await startNode(options, signalHandlers, userLog); } else if (options.pxe) { - // Starting a PXE with a remote node. - // get env vars first - const pxeConfigEnvVars = getPXEServiceConfig(); - // get config from options - const pxeCliOptions = parseModuleOptions(options.pxe); - - // Determine node url from options or env vars - const nodeUrl = pxeCliOptions.nodeUrl || AZTEC_NODE_URL; - // throw if no Aztec Node URL is provided - if (!nodeUrl) { - throw new Error( - 'Aztec Node URL (nodeUrl | AZTEC_NODE_URL) option is required to start PXE without --node option', - ); - } - - // merge env vars and cli options - const pxeConfig = mergeEnvVarsAndCliOptions(pxeConfigEnvVars, pxeCliOptions); - - // create a node client - const node = createAztecNodeClient(nodeUrl); - - const pxe = await createPXEService(node, pxeConfig); - const pxeServer = createPXERpcServer(pxe); - services.push({ pxe: pxeServer }); - signalHandlers.push(pxe.stop); + const { startPXE } = await import('./cmds/start_pxe.js'); + services = await startPXE(options, signalHandlers, userLog); } else if (options.archiver) { - // Start a standalone archiver. - // get env vars first - const archiverConfigEnvVars = getConfigEnvVars(); - // get config from options - const archiverCliOptions = parseModuleOptions(options.archiver); - // merge env vars and cli options - const archiverConfig = mergeEnvVarsAndCliOptions( - archiverConfigEnvVars, - archiverCliOptions, - true, - ); - - const store = await AztecLmdbStore.create( - archiverConfig.l1Contracts.rollupAddress, - archiverConfig.dataDirectory, - ); - const archiverStore = new KVArchiverDataStore(store, archiverConfig.maxLogs); - - const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, true); - const archiverServer = createArchiverRpcServer(archiver); - services.push({ archiver: archiverServer }); - signalHandlers.push(archiver.stop); + const { startArchiver } = await import('./cmds/start_archiver.js'); + await startArchiver(options, signalHandlers); } else if (options.p2pBootstrap) { - // Start a P2P bootstrap node. - const envVars = getP2PConfigEnvVars(); - const cliOptions = parseModuleOptions(options.p2pBootstrap); - const bootstrapNode = new BootstrapNode(debugLogger); - const config = mergeEnvVarsAndCliOptions(envVars, cliOptions); - await bootstrapNode.start(config); - signalHandlers.push(bootstrapNode.stop); + const { startP2PBootstrap } = await import('./cmds/start_p2p_bootstrap.js'); + await startP2PBootstrap(options, signalHandlers, debugLogger); } if (services.length) { const rpcServer = createNamespacedJsonRpcServer(services, debugLogger); diff --git a/yarn-project/aztec/src/cli/cmds/start_archiver.ts b/yarn-project/aztec/src/cli/cmds/start_archiver.ts new file mode 100644 index 00000000000..3eccab217cb --- /dev/null +++ b/yarn-project/aztec/src/cli/cmds/start_archiver.ts @@ -0,0 +1,31 @@ +import { + Archiver, + ArchiverConfig, + KVArchiverDataStore, + createArchiverRpcServer, + getConfigEnvVars as getArchiverConfigEnvVars, +} from '@aztec/archiver'; +import { ServerList } from '@aztec/foundation/json-rpc/server'; +import { AztecLmdbStore } from '@aztec/kv-store'; + +import { mergeEnvVarsAndCliOptions, parseModuleOptions } from '../util.js'; + +export const startArchiver = async (options: any, signalHandlers: (() => Promise)[]) => { + const services: ServerList = []; + // Start a standalone archiver. + // get env vars first + const archiverConfigEnvVars = getArchiverConfigEnvVars(); + // get config from options + const archiverCliOptions = parseModuleOptions(options.archiver); + // merge env vars and cli options + const archiverConfig = mergeEnvVarsAndCliOptions(archiverConfigEnvVars, archiverCliOptions, true); + + const store = await AztecLmdbStore.create(archiverConfig.l1Contracts.rollupAddress, archiverConfig.dataDirectory); + const archiverStore = new KVArchiverDataStore(store, archiverConfig.maxLogs); + + const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, true); + const archiverServer = createArchiverRpcServer(archiver); + services.push({ archiver: archiverServer }); + signalHandlers.push(archiver.stop); + return services; +}; diff --git a/yarn-project/aztec/src/cli/cmds/start_node.ts b/yarn-project/aztec/src/cli/cmds/start_node.ts new file mode 100644 index 00000000000..7a4a0d3623b --- /dev/null +++ b/yarn-project/aztec/src/cli/cmds/start_node.ts @@ -0,0 +1,87 @@ +import { AztecNodeConfig, createAztecNodeRpcServer, getConfigEnvVars as getNodeConfigEnvVars } from '@aztec/aztec-node'; +import { NULL_KEY } from '@aztec/ethereum'; +import { ServerList } from '@aztec/foundation/json-rpc/server'; +import { LogFn } from '@aztec/foundation/log'; +import { PXEServiceConfig, createPXERpcServer, getPXEServiceConfig } from '@aztec/pxe'; + +import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; + +import { MNEMONIC, createAztecNode, createAztecPXE, deployContractsToL1 } from '../../sandbox.js'; +import { mergeEnvVarsAndCliOptions, parseModuleOptions } from '../util.js'; + +const { DEPLOY_AZTEC_CONTRACTS } = process.env; + +export const startNode = async ( + options: any, + signalHandlers: (() => Promise)[], + userLog: LogFn, +): Promise => { + // Services that will be started in a single multi-rpc server + const services: ServerList = []; + // get env vars first + const aztecNodeConfigEnvVars = getNodeConfigEnvVars(); + // get config from options + const nodeCliOptions = parseModuleOptions(options.node); + // merge env vars and cli options + let nodeConfig = mergeEnvVarsAndCliOptions(aztecNodeConfigEnvVars, nodeCliOptions); + + // if no publisher private key, then use MNEMONIC + if (!options.archiver) { + // expect archiver url in node config + const archiverUrl = nodeCliOptions.archiverUrl; + if (!archiverUrl) { + userLog('Archiver Service URL is required to start Aztec Node without --archiver option'); + throw new Error('Archiver Service URL is required to start Aztec Node without --archiver option'); + } + nodeConfig.archiverUrl = archiverUrl; + } else { + const archiverCliOptions = parseModuleOptions(options.archiver); + nodeConfig = mergeEnvVarsAndCliOptions(aztecNodeConfigEnvVars, archiverCliOptions); + } + + // Deploy contracts if needed + if (nodeCliOptions.deployAztecContracts || DEPLOY_AZTEC_CONTRACTS === 'true') { + let account; + if (nodeConfig.publisherPrivateKey === NULL_KEY) { + account = mnemonicToAccount(MNEMONIC); + } else { + account = privateKeyToAccount(nodeConfig.publisherPrivateKey); + } + await deployContractsToL1(nodeConfig, account); + } + + if (!options.sequencer) { + nodeConfig.disableSequencer = true; + } else if (nodeConfig.publisherPrivateKey === NULL_KEY) { + // If we have a sequencer, ensure there's a publisher private key set. + const hdAccount = mnemonicToAccount(MNEMONIC); + const privKey = hdAccount.getHdKey().privateKey; + nodeConfig.publisherPrivateKey = `0x${Buffer.from(privKey!).toString('hex')}`; + } + + // Create and start Aztec Node. + const node = await createAztecNode(nodeConfig); + const nodeServer = createAztecNodeRpcServer(node); + + // Add node to services list + services.push({ node: nodeServer }); + + // Add node stop function to signal handlers + signalHandlers.push(node.stop); + + // Create a PXE client that connects to the node. + if (options.pxe) { + const pxeCliOptions = parseModuleOptions(options.pxe); + const pxeConfig = mergeEnvVarsAndCliOptions(getPXEServiceConfig(), pxeCliOptions); + const pxe = await createAztecPXE(node, pxeConfig); + const pxeServer = createPXERpcServer(pxe); + + // Add PXE to services list + services.push({ pxe: pxeServer }); + + // Add PXE stop function to signal handlers + signalHandlers.push(pxe.stop); + } + + return services; +}; diff --git a/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts b/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts new file mode 100644 index 00000000000..3ed48cf12cf --- /dev/null +++ b/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts @@ -0,0 +1,18 @@ +import { DebugLogger } from '@aztec/aztec.js'; +import { BootstrapNode, P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; + +import { mergeEnvVarsAndCliOptions, parseModuleOptions } from '../util.js'; + +export const startP2PBootstrap = async ( + options: any, + signalHandlers: (() => Promise)[], + debugLogger: DebugLogger, +) => { + // Start a P2P bootstrap node. + const envVars = getP2PConfigEnvVars(); + const cliOptions = parseModuleOptions(options.p2pBootstrap); + const bootstrapNode = new BootstrapNode(debugLogger); + const config = mergeEnvVarsAndCliOptions(envVars, cliOptions); + await bootstrapNode.start(config); + signalHandlers.push(bootstrapNode.stop); +}; diff --git a/yarn-project/aztec/src/cli/cmds/start_pxe.ts b/yarn-project/aztec/src/cli/cmds/start_pxe.ts new file mode 100644 index 00000000000..b97cc431cc8 --- /dev/null +++ b/yarn-project/aztec/src/cli/cmds/start_pxe.ts @@ -0,0 +1,38 @@ +import { createAztecNodeClient } from '@aztec/circuit-types'; +import { ServerList } from '@aztec/foundation/json-rpc/server'; +import { LogFn } from '@aztec/foundation/log'; +import { PXEServiceConfig, createPXERpcServer, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; + +import { mergeEnvVarsAndCliOptions, parseModuleOptions } from '../util.js'; + +const { AZTEC_NODE_URL } = process.env; + +export const startPXE = async (options: any, signalHandlers: (() => Promise)[], userLog: LogFn) => { + // Services that will be started in a single multi-rpc server + const services: ServerList = []; + // Starting a PXE with a remote node. + // get env vars first + const pxeConfigEnvVars = getPXEServiceConfig(); + // get config from options + const pxeCliOptions = parseModuleOptions(options.pxe); + + // Determine node url from options or env vars + const nodeUrl = pxeCliOptions.nodeUrl || AZTEC_NODE_URL; + // throw if no Aztec Node URL is provided + if (!nodeUrl) { + userLog('Aztec Node URL (nodeUrl | AZTEC_NODE_URL) option is required to start PXE without --node option'); + throw new Error('Aztec Node URL (nodeUrl | AZTEC_NODE_URL) option is required to start PXE without --node option'); + } + + // merge env vars and cli options + const pxeConfig = mergeEnvVarsAndCliOptions(pxeConfigEnvVars, pxeCliOptions); + + // create a node client + const node = createAztecNodeClient(nodeUrl); + + const pxe = await createPXEService(node, pxeConfig); + const pxeServer = createPXERpcServer(pxe); + services.push({ pxe: pxeServer }); + signalHandlers.push(pxe.stop); + return services; +}; diff --git a/yarn-project/aztec/src/cli/texts.ts b/yarn-project/aztec/src/cli/texts.ts index 38cdbb57f06..7224011e76b 100644 --- a/yarn-project/aztec/src/cli/texts.ts +++ b/yarn-project/aztec/src/cli/texts.ts @@ -26,9 +26,10 @@ export const cliTexts = { 'rcpUrl:ETHEREUM_HOST - string - The host of the Ethereum node to connect to. Default: http://localhost:8545\n' + 'archiverUrl:ARCHIVER_URL - string - A URL for an archiver service that the node will use.\n' + 'p2pEnabled:P2P_ENABLED - boolean - A flag dictating whether the P2P subsystem should be enabled.\n\n' + + 'dataDirectory:DATA_DIRECTORY - string - Where to store node data. If not set, will store temporarily.\n', 'deployAztecContracts:DEPLOY_AZTEC_CONTRACTS - boolean - A flag dictating whether to deploy the Aztec contracts. Default: false\n' + 'l2QueueSize:L2_QUEUE_SIZE - number - Size of queue of L2 blocks to store. Default: 1000\n' + - 'worldStateBlockCheckIntervalMS:WS_BLOCK_CHECK_INTERVAL_MS - number - The frequency in which to check for blocks in ms. Default: 100\n\n' + + 'worldStateBlockCheckIntervalMS:WS_BLOCK_CHECK_INTERVAL_MS - number - The frequency in which to check for blocks in ms. Default: 100\n' + // Contract Addresses contractAddresses + // P2P Options @@ -41,7 +42,7 @@ export const cliTexts = { 'port:PXE_PORT - number - The port on which the PXE should listen for connections. Default: 79\n' + 'l2BlockPollingIntervalMS:PXE_BLOCK_POLLING_INTERVAL_MS - number - The frequency in which to check for blocks in ms. Default: 1000\n' + 'l2StartingBlock:PXE_L2_STARTING_BLOCK - number - The block number from which to start polling. Default: 1\n' + - 'dataDirectory:PXE_DATA_DIRECTORY - string - Where to store PXE data. If not set, will store in memory.\n', + 'dataDirectory:PXE_DATA_DIRECTORY - string - Where to store PXE data. If not set, will store temporarily.\n', archiver: 'Starts an Archiver with options. If started additionally to --node, the Archiver will attach to that node.' + 'Available options are listed below as cliProperty:ENV_VARIABLE_NAME.\n' + @@ -49,7 +50,7 @@ export const cliTexts = { 'apiKey:API_KEY - string - The key for the ethereum node if necessary.\n' + 'archiverPollingIntervalMS:ARCHIVER_POLLING_INTERVAL_MS - number - The polling interval in ms for retrieving new L2 blocks and encrypted logs. Default: 1000\n' + 'viemPollingIntervalMS:ARCHIVER_VIEM_POLLING_INTERVAL_MS - number - The polling interval viem uses in ms. Default: 1000\n' + - 'dataDirectory:DATA_DIRECTORY - string - Optional dir to store data. If omitted will store in memory.\n\n' + + 'dataDirectory:DATA_DIRECTORY - string - Optional dir to store data. If omitted will store temporarily.\n\n' + contractAddresses, sequencer: 'Starts a Sequencer with options. If started additionally to --node, the Sequencer will attach to that node.\n' + @@ -61,7 +62,7 @@ export const cliTexts = { 'publisherPrivateKey:SEQ_PUBLISHER_PRIVATE_KEY - string - The private key of the publisher. If not provided, will try to infer from default foundry test accounts.\n' + 'requiredConfirmations:SEQ_REQUIRED_CONFIRMATIONS - number - The number of confirmations required before publishing a block. Default: 1\n' + 'l1BlockPublishRetryIntervalMS:SEQ_PUBLISH_RETRY_INTERVAL_MS - number - The interval in ms to wait before retrying to publish a block. Default: 1000\n' + - 'transactionPollingIntervalMS:SEQ_TX_POLLING_INTERVAL_MS - number - The interval in ms to wait before polling for new transactions. Default: 1000\n\n' + + 'transactionPollingIntervalMS:SEQ_TX_POLLING_INTERVAL_MS - number - The interval in ms to wait before polling for new transactions. Default: 1000\n' + contractAddresses, p2pBootstrap: 'Starts a P2P bootstrap node with options.\n' + From bca525b477d38d828a5ef753e46134d91383c169 Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 25 Jan 2024 16:15:26 +0000 Subject: [PATCH 48/50] fix typo --- yarn-project/aztec/src/cli/texts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/aztec/src/cli/texts.ts b/yarn-project/aztec/src/cli/texts.ts index 7224011e76b..2536e37637d 100644 --- a/yarn-project/aztec/src/cli/texts.ts +++ b/yarn-project/aztec/src/cli/texts.ts @@ -26,7 +26,7 @@ export const cliTexts = { 'rcpUrl:ETHEREUM_HOST - string - The host of the Ethereum node to connect to. Default: http://localhost:8545\n' + 'archiverUrl:ARCHIVER_URL - string - A URL for an archiver service that the node will use.\n' + 'p2pEnabled:P2P_ENABLED - boolean - A flag dictating whether the P2P subsystem should be enabled.\n\n' + - 'dataDirectory:DATA_DIRECTORY - string - Where to store node data. If not set, will store temporarily.\n', + 'dataDirectory:DATA_DIRECTORY - string - Where to store node data. If not set, will store temporarily.\n' + 'deployAztecContracts:DEPLOY_AZTEC_CONTRACTS - boolean - A flag dictating whether to deploy the Aztec contracts. Default: false\n' + 'l2QueueSize:L2_QUEUE_SIZE - number - Size of queue of L2 blocks to store. Default: 1000\n' + 'worldStateBlockCheckIntervalMS:WS_BLOCK_CHECK_INTERVAL_MS - number - The frequency in which to check for blocks in ms. Default: 100\n' + From 72139a2228ac4ac4d5fd8a46ec28e63ff8fc0342 Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 26 Jan 2024 16:46:37 +0000 Subject: [PATCH 49/50] update lmdb store method --- yarn-project/aztec/src/cli/cmds/start_archiver.ts | 2 +- yarn-project/p2p/src/client/p2p_client.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn-project/aztec/src/cli/cmds/start_archiver.ts b/yarn-project/aztec/src/cli/cmds/start_archiver.ts index 3eccab217cb..0a07c96ae6c 100644 --- a/yarn-project/aztec/src/cli/cmds/start_archiver.ts +++ b/yarn-project/aztec/src/cli/cmds/start_archiver.ts @@ -20,7 +20,7 @@ export const startArchiver = async (options: any, signalHandlers: (() => Promise // merge env vars and cli options const archiverConfig = mergeEnvVarsAndCliOptions(archiverConfigEnvVars, archiverCliOptions, true); - const store = await AztecLmdbStore.create(archiverConfig.l1Contracts.rollupAddress, archiverConfig.dataDirectory); + const store = await AztecLmdbStore.open(archiverConfig.l1Contracts.rollupAddress, archiverConfig.dataDirectory); const archiverStore = new KVArchiverDataStore(store, archiverConfig.maxLogs); const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, true); diff --git a/yarn-project/p2p/src/client/p2p_client.ts b/yarn-project/p2p/src/client/p2p_client.ts index 3d85307eb95..8fb1b520cb0 100644 --- a/yarn-project/p2p/src/client/p2p_client.ts +++ b/yarn-project/p2p/src/client/p2p_client.ts @@ -132,8 +132,8 @@ export class P2PClient implements P2P { private p2pService: P2PService, private log = createDebugLogger('aztec:p2p'), ) { - const { p2pBlockCheckIntervalMS: checkInterval, l2QueueSize } = getP2PConfigEnvVars(); - this.blockDownloader = new L2BlockDownloader(l2BlockSource, l2QueueSize, checkInterval); + const { p2pBlockCheckIntervalMS: checkInterval, p2pL2QueueSize } = getP2PConfigEnvVars(); + this.blockDownloader = new L2BlockDownloader(l2BlockSource, p2pL2QueueSize, checkInterval); this.synchedBlockNumber = store.openSingleton('p2p_pool_last_l2_block'); } From 1bd17006ba1f55b7ef82d7fee6fca9e61a2249fc Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 26 Jan 2024 18:01:21 +0000 Subject: [PATCH 50/50] remove unused packages from aztec --- yarn-project/aztec/package.json | 3 --- yarn-project/yarn.lock | 3 --- 2 files changed, 6 deletions(-) diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index f1b3cc41959..fbc0b5dcb20 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -45,9 +45,6 @@ "commander": "^11.1.0", "koa": "^2.14.2", "koa-router": "^12.0.0", - "leveldown": "^6.1.1", - "lmdb": "^2.9.2", - "memdown": "^6.1.1", "viem": "^1.2.5", "winston": "^3.10.0", "winston-daily-rotate-file": "^4.7.1" diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index d4fb1c68998..88e50f74f76 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -274,9 +274,6 @@ __metadata: jest: ^29.5.0 koa: ^2.14.2 koa-router: ^12.0.0 - leveldown: ^6.1.1 - lmdb: ^2.9.2 - memdown: ^6.1.1 ts-jest: ^29.1.0 ts-node: ^10.9.1 typescript: ^5.0.4