feat: multiple pixies 1 Sandbox (#2492)

1. Fixes #2460 2. Sneaked in unrelated name change `PXE_SERVICE_BLOCK_POLLING_INTERVAL_MS` -> `PXE_BLOCK_POLLING_INTERVAL_MS`. **Note 1**: I created a separate CI task for deploying docker image of pxe so if that task or image is messed up it shouldn't be that big of a deal because it would not affect the sandbox image. **Note 2**: Instead of `SANDBOX_URL` env var there are now `PXE_URL` and `AZTEC_NODE_URL`. The e2e test automatically uses sandbox if `PXE_URL` is set. If `PXE_URL` is set and `AZTEC_NODE_URL` is not the e2e tests will try to create a client on the same host as PXE but with a port 8079. **Note 3**: I needed to expose more methods of the AztecNode via RPC and to do that I had to implement serialization functionality of L1 message with index type.
AztecProtocol · Oct 4, 2023 · 572d572 · 572d572
1 parent c484799
commit 572d572
Show file tree

Hide file tree

Showing 69 changed files with 446 additions and 227 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -454,7 +454,44 @@ jobs:
             echo "export DOCKER_BUILDKIT=" > $BASH_ENV
             force_deploy_build aztec-sandbox false arm64
 
-  aztec-sandbox-ecr-manifest:
+  pxe-base:
+    machine:
+      image: ubuntu-2204:2023.07.2
+    resource_class: large
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Build and test"
+          command: force_deploy_build pxe-base false
+
+  pxe-x86_64:
+    machine:
+      image: ubuntu-2204:2023.07.2
+    resource_class: large
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Build and test"
+          command: force_deploy_build pxe false x86_64
+
+  pxe-arm64:
+    machine:
+      image: ubuntu-2204:2023.07.2
+    resource_class: arm.large
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Build and test"
+          # We need to force not to use docker buildkit because for some reason on arm only, it ends up making a call
+          # out to eu-west2 despite the image being locally tagged, resulting in unauthorised 401. Weird docker bug?
+          command: |
+            echo "export DOCKER_BUILDKIT=" > $BASH_ENV
+            force_deploy_build pxe false arm64
+
+  ecr-manifest:
     machine:
       image: ubuntu-2204:2023.07.2
     resource_class: large
@@ -463,7 +500,9 @@ jobs:
       - *setup_env
       - run:
           name: "Create ECR manifest"
-          command: create_ecr_manifest aztec-sandbox-base aztec-sandbox x86_64,arm64
+          command: |
+            create_ecr_manifest aztec-sandbox-base aztec-sandbox x86_64,arm64
+            create_ecr_manifest pxe-base pxe x86_64,arm64
 
   boxes-blank-react:
     machine:
@@ -993,7 +1032,7 @@ jobs:
           name: "yarn-project"
           command: yarn-project/deploy_npm.sh
 
-  deploy-dockerhub:
+  deploy-dockerhub-sandbox:
     machine:
       image: ubuntu-2204:2023.07.2
     resource_class: medium
@@ -1008,6 +1047,21 @@ jobs:
             deploy_dockerhub aztec-sandbox arm64
             create_dockerhub_manifest aztec-sandbox x86_64,arm64
 
+  deploy-dockerhub-pxe:
+    machine:
+      image: ubuntu-2204:2023.07.2
+    resource_class: medium
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "deploy-pxe"
+          working_directory: pxe
+          command: |
+            deploy_dockerhub pxe x86_64
+            deploy_dockerhub pxe arm64
+            create_dockerhub_manifest pxe x86_64,arm64
+
   deploy-end:
     docker:
       - image: cimg/base:2023.09
@@ -1201,6 +1255,7 @@ workflows:
 
       - end-to-end: *yarn_project
       - aztec-sandbox-base: *yarn_project
+      - pxe-base: *yarn_project
       - canary: *yarn_project
       - build-docs: *yarn_project
 
@@ -1213,10 +1268,21 @@ workflows:
             - aztec-sandbox-base
           <<: *defaults
 
-      - aztec-sandbox-ecr-manifest:
+      - pxe-x86_64:
+          requires:
+            - pxe-base
+          <<: *defaults
+      - pxe-arm64:
+          requires:
+            - pxe-base
+          <<: *defaults
+
+      - ecr-manifest:
           requires:
             - aztec-sandbox-x86_64
             - aztec-sandbox-arm64
+            - pxe-x86_64
+            - pxe-arm64
           <<: *defaults
 
       - boxes-blank-react:
@@ -1237,7 +1303,7 @@ workflows:
       - e2e-join:
           requires:
             - end-to-end
-            - aztec-sandbox-ecr-manifest
+            - ecr-manifest
             - canary
           <<: *defaults
 
@@ -1322,7 +1388,11 @@ workflows:
           <<: *defaults
 
         # Deployment and Canary tests
-      - deploy-dockerhub:
+      - deploy-dockerhub-sandbox:
+          requires:
+            - e2e-end
+          <<: *deploy_defaults
+      - deploy-dockerhub-pxe:
           requires:
             - e2e-end
           <<: *deploy_defaults
@@ -1333,7 +1403,8 @@ workflows:
 
       - deploy-end:
           requires:
-            - deploy-dockerhub
+            - deploy-dockerhub-sandbox
+            - deploy-dockerhub-pxe
             - deploy-npm
           <<: *deploy_defaults
 

diff --git a/build_manifest.yml b/build_manifest.yml
@@ -120,6 +120,20 @@ aztec-sandbox:
   dependencies:
     - aztec-sandbox-base
 
+pxe-base:
+  buildDir: yarn-project
+  projectDir: yarn-project/pxe
+  dockerfile: Dockerfile
+  dependencies:
+    - yarn-project
+
+pxe:
+  buildDir: yarn-project
+  projectDir: yarn-project/pxe
+  dockerfile: Dockerfile.final
+  dependencies:
+    - pxe-base
+
 boxes-blank-react:
   buildDir: yarn-project
   projectDir: yarn-project/boxes/blank-react

diff --git a/docs/docs/dev_docs/cli/main.md b/docs/docs/dev_docs/cli/main.md
@@ -36,7 +36,7 @@ Once installed it is invoked via:
 
 ## I have the Sandbox running, now what?
 
-Lets first establish that we are able to communicate with the Sandbox. Most commands will require the url to the Sandbox, which defaults in the CLI to `http://localhost:8080`. You can override this as an option with each command or by setting `PXE_HOST` environment variable.
+Lets first establish that we are able to communicate with the Sandbox. Most commands will require the url to the Sandbox, which defaults in the CLI to `http://localhost:8080`. You can override this as an option with each command or by setting `PXE_URL` environment variable.
 
 To test communication with the Sandbox, let's run the command:
 

diff --git a/docs/docs/dev_docs/getting_started/sandbox.md b/docs/docs/dev_docs/getting_started/sandbox.md
@@ -36,7 +36,11 @@ It will download and execute a script invoking docker compose with 2 containers:
 - Anvil
 - Aztec Sandbox
 
-2 ports will need to be opened on your system in order for you to interact with the sandbox. The first port is for Anvil, it defaults to 8545 and can be overridden by specifying a value in the environment variable `SANDBOX_ANVIL_PORT`. The second is the sandbox RPC host port. It defaults to value 8080 but can be overridden with environment variable `SANDBOX_RPC_PORT`.
+3 ports will need to be opened on your system in order for you to interact with the sandbox.
+The first port is for Anvil, it defaults to 8545 and can be overridden by specifying a value in the environment variable `SANDBOX_ANVIL_PORT`.
+The second one is sandbox Aztec Node port, it defaults to 8079 and can be overridden by specifying a value in the environment variable `SANDBOX_AZTEC_NODE_PORT`.
+The third is the sandbox PXE port.
+It defaults to value 8080 but can be overridden with environment variable `SANDBOX_PXE_PORT`.
 
 Within a few seconds the Sandbox should be up and running!
 

diff --git a/yarn-project/aztec-node/src/aztec-node/http_rpc_server.ts b/yarn-project/aztec-node/src/aztec-node/http_rpc_server.ts
@@ -3,7 +3,18 @@ import { AztecAddress } from '@aztec/foundation/aztec-address';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { Fr } from '@aztec/foundation/fields';
 import { JsonRpcServer } from '@aztec/foundation/json-rpc/server';
-import { AztecNode, ContractData, ExtendedContractData, L2Block, L2BlockL2Logs, L2Tx, Tx, TxHash } from '@aztec/types';
+import {
+  AztecNode,
+  ContractData,
+  ExtendedContractData,
+  L1ToL2MessageAndIndex,
+  L2Block,
+  L2BlockL2Logs,
+  L2Tx,
+  SiblingPath,
+  Tx,
+  TxHash,
+} from '@aztec/types';
 
 /**
  * Wrap an AztecNode instance with a JSON RPC HTTP server.
@@ -13,11 +24,23 @@ import { AztecNode, ContractData, ExtendedContractData, L2Block, L2BlockL2Logs,
 export function createAztecNodeRpcServer(node: AztecNode) {
   const rpc = new JsonRpcServer(
     node,
-    { AztecAddress, EthAddress, ExtendedContractData, ContractData, Fr, HistoricBlockData, L2Block, L2Tx, TxHash },
+    {
+      AztecAddress,
+      EthAddress,
+      ExtendedContractData,
+      ContractData,
+      Fr,
+      HistoricBlockData,
+      L2Block,
+      L2Tx,
+      TxHash,
+      SiblingPath,
+      L1ToL2MessageAndIndex,
+    },
     { Tx, L2BlockL2Logs },
     false,
     // disable methods not part of the AztecNode interface
-    ['start', 'stop', 'getDataTreePath', 'getL1ToL2MessageAndIndex', 'getL1ToL2MessagesTreePath'],
+    ['start', 'stop'],
   );
   return rpc;
 }
diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts
@@ -299,7 +299,7 @@ export class AztecNodeService implements AztecNode {
     // todo: #697 - make this one lookup.
     const index = (await this.findLeafIndex(MerkleTreeId.L1_TO_L2_MESSAGES_TREE, messageKey.toBuffer()))!;
     const message = await this.l1ToL2MessageSource.getConfirmedL1ToL2Message(messageKey);
-    return Promise.resolve({ message, index });
+    return Promise.resolve(new L1ToL2MessageAndIndex(index, message));
   }
 
   /**

diff --git a/yarn-project/aztec-node/src/bin/index.ts b/yarn-project/aztec-node/src/bin/index.ts
@@ -7,7 +7,7 @@ import Router from 'koa-router';
 
 import { AztecNodeConfig, AztecNodeService, createAztecNodeRpcServer, getConfigEnvVars } from '../index.js';
 
-const { SERVER_PORT = 8081, API_PREFIX = '' } = process.env;
+const { AZTEC_NODE_PORT = 8081, API_PREFIX = '' } = process.env;
 
 const logger = createDebugLogger('aztec:node');
 
@@ -57,8 +57,8 @@ async function main() {
   app.use(apiRouter.allowedMethods());
 
   const httpServer = http.createServer(app.callback());
-  httpServer.listen(+SERVER_PORT);
-  logger.info(`Aztec Node JSON-RPC Server listening on port ${SERVER_PORT}`);
+  httpServer.listen(+AZTEC_NODE_PORT);
+  logger.info(`Aztec Node JSON-RPC Server listening on port ${AZTEC_NODE_PORT}`);
 }
 
 main().catch(err => {

diff --git a/yarn-project/aztec-node/terraform/main.tf b/yarn-project/aztec-node/terraform/main.tf
@@ -98,7 +98,7 @@ resource "aws_ecs_task_definition" "aztec-node-1" {
         "value": "production"
       },
       {
-        "name": "SERVER_PORT",
+        "name": "AZTEC_NODE_PORT",
         "value": "80"
       },
       {

diff --git a/yarn-project/aztec-sandbox/Dockerfile b/yarn-project/aztec-sandbox/Dockerfile
@@ -22,4 +22,4 @@ COPY --from=builder /usr/src/ /usr/src/
 WORKDIR /usr/src/yarn-project/aztec-sandbox
 ENTRYPOINT ["yarn"]
 CMD [ "start" ]
-EXPOSE 8080
+EXPOSE 8079 8080
diff --git a/yarn-project/aztec-sandbox/Dockerfile.final b/yarn-project/aztec-sandbox/Dockerfile.final
@@ -7,4 +7,4 @@ WORKDIR /usr/src/yarn-project/aztec-sandbox
 
 ENTRYPOINT ["yarn"]
 CMD [ "start" ]
-EXPOSE 8080
+EXPOSE 8079 8080
diff --git a/yarn-project/aztec-sandbox/docker-compose.yml b/yarn-project/aztec-sandbox/docker-compose.yml
@@ -9,7 +9,8 @@ services:
   aztec:
     image: 'aztecprotocol/aztec-sandbox:${SANDBOX_VERSION:-latest}'
     ports:
-      - '${SANDBOX_RPC_PORT:-8080}:8080'
+      - '${SANDBOX_AZTEC_NODE_PORT:-8079}:8079'
+      - '${SANDBOX_PXE_PORT:-8080}:8080'
     environment:
       DEBUG: # Loaded from the user shell if explicitly set
       HOST_WORKDIR: '${PWD}' # Loaded from the user shell to show log files absolute path in host
@@ -19,7 +20,7 @@ services:
       P2P_BLOCK_CHECK_INTERVAL_MS: 50
       SEQ_TX_POLLING_INTERVAL_MS: 50
       WS_BLOCK_CHECK_INTERVAL_MS: 50
-      PXE_SERVICE_BLOCK_POLLING_INTERVAL_MS: 50
+      PXE_BLOCK_POLLING_INTERVAL_MS: 50
       ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500
     volumes:
       - ./log:/usr/src/yarn-project/aztec-sandbox/log:rw
diff --git a/yarn-project/aztec-sandbox/src/bin/index.ts b/yarn-project/aztec-sandbox/src/bin/index.ts
@@ -1,31 +1,34 @@
 #!/usr/bin/env -S node --no-warnings
+import { createAztecNodeRpcServer } from '@aztec/aztec-node';
 import { deployInitialSandboxAccounts } from '@aztec/aztec.js';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { fileURLToPath } from '@aztec/foundation/url';
 import NoirVersion from '@aztec/noir-compiler/noir-version';
-import { startPXEHttpServer } from '@aztec/pxe';
+import { createPXERpcServer } from '@aztec/pxe';
 
 import { readFileSync } from 'fs';
 import { dirname, resolve } from 'path';
 
 import { setupFileDebugLog } from '../logging.js';
 import { createSandbox } from '../sandbox.js';
+import { startHttpRpcServer } from '../server.js';
 import { github, splash } from '../splash.js';
 
-const { SERVER_PORT = 8080 } = process.env;
+const { AZTEC_NODE_PORT = 8079, PXE_PORT = 8080 } = process.env;
 
 const logger = createDebugLogger('aztec:sandbox');
 
 /**
  * Creates the sandbox from provided config and deploys any initial L1 and L2 contracts
  */
 async function createAndInitialiseSandbox() {
-  const { l1Contracts, pxe, stop } = await createSandbox();
+  const { l1Contracts, node, pxe, stop } = await createSandbox();
   logger.info('Setting up test accounts...');
   const accounts = await deployInitialSandboxAccounts(pxe);
   return {
     l1Contracts,
     pxe,
+    node,
     stop,
     accounts,
   };
@@ -41,7 +44,7 @@ async function main() {
 
   logger.info(`Setting up Aztec Sandbox v${version} (nargo ${NoirVersion.tag}), please stand by...`);
 
-  const { pxe, stop, accounts } = await createAndInitialiseSandbox();
+  const { pxe, node, stop, accounts } = await createAndInitialiseSandbox();
 
   const shutdown = async () => {
     logger.info('Shutting down...');
@@ -52,8 +55,10 @@ async function main() {
   process.once('SIGINT', shutdown);
   process.once('SIGTERM', shutdown);
 
-  startPXEHttpServer(pxe, SERVER_PORT);
-  logger.info(`Aztec Sandbox JSON-RPC Server listening on port ${SERVER_PORT}`);
+  startHttpRpcServer(node, createAztecNodeRpcServer, AZTEC_NODE_PORT);
+  logger.info(`Aztec Node JSON-RPC Server listening on port ${AZTEC_NODE_PORT}`);
+  startHttpRpcServer(pxe, createPXERpcServer, PXE_PORT);
+  logger.info(`PXE JSON-RPC Server listening on port ${PXE_PORT}`);
   logger.info(`Debug logs will be written to ${logPath}`);
   const accountStrings = [`Initial Accounts:\n\n`];
 

diff --git a/yarn-project/aztec-sandbox/src/routes.ts b/yarn-project/aztec-sandbox/src/routes.ts