chore(blobs): blob sink in sandbox and native network tests

.github/workflows/publish-aztec-packages.yml

@@ -183,6 +183,36 @@ jobs:
         run: |
           earthly-ci --no-output --push ./yarn-project+export-cli-wallet --DIST_TAG=${{ env.GIT_COMMIT }} --ARCH=arm64
 
+  build-blob-sink-x86:
+    needs: [configure, build-aztec-x86]
+    runs-on: ${{ needs.configure.outputs.username }}-x86
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: "${{ env.GIT_COMMIT }}"
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: build-aztec-blob-sink-x86_64
+          dockerhub_password: "${{ env.DOCKERHUB_PASSWORD }}"
+      - name: Build & push aztec blob sink image x86-64
+        run: |
+          earthly-ci --no-output --push ./yarn-project+export-blob-sink --DIST_TAG=${{ env.GIT_COMMIT }} --ARCH=x86_64
+
+  build-blob-sink-arm:
+    needs: [configure, build-aztec-arm]
+    runs-on: ${{ needs.configure.outputs.username }}-arm
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: "${{ env.GIT_COMMIT }}"
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: build-aztec-blob-sink-arm64
+          dockerhub_password: "${{ env.DOCKERHUB_PASSWORD }}"
+      - name: Build & push aztec blob sink image arm
+        run: |
+          earthly-ci --no-output --push ./yarn-project+export-blob-sink --DIST_TAG=${{ env.GIT_COMMIT }} --ARCH=arm64
+
   publish-manifests:
     needs:
       - configure

aztec-up/bin/aztec-install

@@ -107,6 +107,7 @@ if [ -z "${SKIP_PULL:-}" ]; then
   pull_container aztec-nargo
   pull_container aztec
   pull_container cli-wallet
+  pull_container blob-sink
 fi
 
 # Download the Docker Compose file. Used by aztec.

aztec-up/bin/docker-compose.sandbox.yml

@@ -15,6 +15,13 @@ services:
       FORK_BLOCK_NUMBER:
       ANVIL_PORT: ${ANVIL_PORT:-8545}
 
+  # TODO: add a readiness probe to this
+  # TODO: delete all of these extra images and just have one, then symlink them???
+  blob-sink:
+    image: aztecprotocol/blob-sink
+    ports:
+      - "${BLOB_SINK_PORT:-5052}:${BLOB_SINK_PORT:-5052}"
+
   aztec:
     image: "aztecprotocol/aztec"
     ports:
@@ -29,11 +36,13 @@ services:
       SEQ_TX_POLLING_INTERVAL_MS: 50
       WS_BLOCK_CHECK_INTERVAL_MS: 50
       ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500
+      SEQ_BLOB_SINK_URL: http://blob-sink:${BLOB_SINK_PORT:-5052}
       PXE_PORT: ${PXE_PORT:-8080}
       PORT: ${AZTEC_NODE_PORT:-8080}
       TEST_ACCOUNTS: ${TEST_ACCOUNTS:-true}
     volumes:
       - ./log:/usr/src/yarn-project/aztec/log:rw
     depends_on:
       - ethereum
+      - blob-sink
     command: "start --sandbox"

boxes/docker-compose.yml

@@ -25,6 +25,7 @@ services:
       SEQ_TX_POLLING_INTERVAL_MS: 50
       WS_BLOCK_CHECK_INTERVAL_MS: 50
       ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500
+      SEQ_BLOB_SINK_URL: http://blob-sink:5052
     healthcheck:
       test: ["CMD", "curl", "-fSs", "http://127.0.0.1:8080/status"]
       interval: 3s
@@ -33,6 +34,13 @@ services:
     depends_on:
       - ethereum
 
+  blob-sink:
+    image: aztecprotocol/blob-sink:${AZTEC_DOCKER_TAG:-latest}
+    environment:
+      PORT: 5052
+      DEBUG: "aztec:*"
+      DEBUG_COLORS: "true"
+
   boxes:
     image: aztecprotocol/build:2.0
     tty: true

build_manifest.yml

@@ -242,6 +242,13 @@ cli-wallet:
     - yarn-project
   multiarch: buildx
 
+blob-sink:
+  buildDir: yarn-project
+  projectDir: yarn-project/blob-sink
+  dependencies:
+    - yarn-project
+  multiarch: buildx
+
 # Builds all the boxes. They are then independently tested in the container.
 boxes:
   buildDir: boxes

scripts/run_native_testnet.sh

@@ -128,7 +128,8 @@ BASE_CMD="INTERLEAVED=$INTERLEAVED ./yarn-project/end-to-end/scripts/native_netw
         \"./validators.sh $NUM_VALIDATORS\" \
         $PROVER_SCRIPT \
         ./pxe.sh \
-        ./transaction-bot.sh"
+        ./transaction-bot.sh \
+        ./blob-sink.sh"
 
 # Execute the command
 eval $BASE_CMD

yarn-project/Earthfile

@@ -22,6 +22,24 @@ export-cli-wallet:
     ARG ARCH
     SAVE IMAGE --push aztecprotocol/cli-wallet:${DIST_TAG}${ARCH:+-$ARCH}
 
+blob-sink-build:
+    FROM +bootstrap
+    RUN yarn workspaces focus @aztec/blob-sink --production && yarn cache clean
+    SAVE ARTIFACT /usr/src /usr/src
+
+blob-sink:
+    FROM ubuntu:noble
+    RUN apt update && apt install nodejs curl -y && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+    COPY +blob-sink-build/usr/src /usr/src
+    ENTRYPOINT ["node", "/usr/src/yarn-project/blob-sink/dest/run.js"]
+
+# TODO(ci3) move to bootstrap.sh
+export-blob-sink:
+    FROM +blob-sink
+    ARG DIST_TAG="latest"
+    ARG ARCH
+    SAVE IMAGE --push aztecprotocol/blob-sink:${DIST_TAG}${ARCH:+-$ARCH}
+
 anvil:
     FROM ../build-images+from-registry
     SAVE ARTIFACT /opt/foundry/bin/anvil

yarn-project/blob-sink/.eslintrc.cjs

@@ -0,0 +1 @@
+module.exports = require('@aztec/foundation/eslint');

yarn-project/blob-sink/Dockerfile

@@ -0,0 +1,7 @@
+## TODO THIS
+
+FROM aztecprotocol/yarn-project AS yarn-project
+
+RUN apt update
+
+ENTRYPOINT ["node", "/usr/src/yarn-project/blob-sink/dest/bin/run.js"]

yarn-project/blob-sink/README.md

@@ -0,0 +1,7 @@
+## Blob Sink
+
+A HTTP api that emulated the https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars API.
+
+## When is this used?
+
+This service will run alongside end to end tests to capture the blob transactions that are sent alongside a `propose` transaction.

yarn-project/blob-sink/package.json

@@ -0,0 +1,155 @@
+{
+  "name": "@aztec/blob-sink",
+  "version": "0.1.0",
+  "type": "module",
+  "exports": {
+    ".": "./dest/index.js"
+  },
+  "inherits": [
+    "../package.common.json"
+  ],
+  "scripts": {
+    "build": "yarn clean && tsc -b",
+    "build:dev": "tsc -b --watch",
+    "clean": "rm -rf ./dest .tsbuildinfo",
+    "formatting": "run -T prettier --check ./src && run -T eslint ./src",
+    "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
+    "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
+  },
+  "jest": {
+    "moduleNameMapper": {
+      "^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
+    },
+    "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$",
+    "rootDir": "./src",
+    "transform": {
+      "^.+\\.tsx?$": [
+        "@swc/jest",
+        {
+          "jsc": {
+            "parser": {
+              "syntax": "typescript",
+              "decorators": true
+            },
+            "transform": {
+              "decoratorVersion": "2022-03"
+            }
+          }
+        }
+      ]
+    },
+    "extensionsToTreatAsEsm": [
+      ".ts"
+    ],
+    "scripts": {
+      "build": "yarn clean && tsc -b",
+      "start": "node ./dest/bin/run.js",
+      "build:dev": "tsc -b --watch",
+      "clean": "rm -rf ./dest .tsbuildinfo",
+      "formatting": "run -T prettier --check ./src && run -T eslint ./src",
+      "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
+      "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests"
+    },
+    "jest": {
+      "moduleNameMapper": {
+        "^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
+      },
+      "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$",
+      "rootDir": "./src",
+      "transform": {
+        "^.+\\.tsx?$": [
+          "@swc/jest",
+          {
+            "jsc": {
+              "parser": {
+                "syntax": "typescript",
+                "decorators": true
+              },
+              "transform": {
+                "decoratorVersion": "2022-03"
+              }
+            }
+          }
+        ]
+      },
+      "extensionsToTreatAsEsm": [
+        ".ts"
+      ],
+      "reporters": [
+        [
+          "default",
+          {
+            "summaryThreshold": 9999
+          }
+        ]
+      ]
+    },
+    "dependencies": {
+      "@aztec/circuit-types": "workspace:^",
+      "@aztec/foundation": "workspace:^",
+      "@aztec/kv-store": "workspace:*",
+      "@aztec/telemetry-client": "workspace:*",
+      "express": "^4.21.1",
+      "source-map-support": "^0.5.21",
+      "tslib": "^2.4.0",
+      "zod": "^3.23.8"
+    },
+    "devDependencies": {
+      "@jest/globals": "^29.5.0",
+      "@types/jest": "^29.5.0",
+      "@types/memdown": "^3.0.0",
+      "@types/node": "^18.7.23",
+      "@types/source-map-support": "^0.5.10",
+      "@types/supertest": "^6.0.2",
+      "jest": "^29.5.0",
+      "jest-mock-extended": "^3.0.3",
+      "supertest": "^7.0.0",
+      "ts-node": "^10.9.1",
+      "typescript": "^5.0.4"
+    },
+    "files": [
+      "dest",
+      "src",
+      "!*.test.*"
+    ],
+    "reporters": [
+      "default"
+    ],
+    "testTimeout": 30000,
+    "setupFiles": [
+      "../../foundation/src/jest/setup.mjs"
+    ]
+  },
+  "dependencies": {
+    "@aztec/circuit-types": "workspace:^",
+    "@aztec/foundation": "workspace:^",
+    "@aztec/kv-store": "workspace:*",
+    "@aztec/telemetry-client": "workspace:*",
+    "express": "^4.21.1",
+    "source-map-support": "^0.5.21",
+    "tslib": "^2.4.0",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.5.0",
+    "@types/jest": "^29.5.0",
+    "@types/memdown": "^3.0.0",
+    "@types/node": "^18.7.23",
+    "@types/source-map-support": "^0.5.10",
+    "@types/supertest": "^6.0.2",
+    "jest": "^29.5.0",
+    "jest-mock-extended": "^3.0.3",
+    "supertest": "^7.0.0",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.0.4"
+  },
+  "files": [
+    "dest",
+    "src",
+    "!*.test.*"
+  ],
+  "types": "./dest/index.d.ts",
+  "engines": {
+    "node": ">=18"
+  }
+}