feat: add blob sink to docker compose and sandbox

.github/workflows/publish-aztec-packages.yml

@@ -183,6 +183,36 @@ jobs:
         run: |
           earthly-ci --no-output --push ./yarn-project+export-cli-wallet --DIST_TAG=${{ env.GIT_COMMIT }} --ARCH=arm64
 
+  build-blob-sink-x86:
+    needs: [configure, build-aztec-x86]
+    runs-on: ${{ needs.configure.outputs.username }}-x86
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: "${{ env.GIT_COMMIT }}"
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: build-aztec-blob-sink-x86_64
+          dockerhub_password: "${{ env.DOCKERHUB_PASSWORD }}"
+      - name: Build & push aztec blob sink image x86-64
+        run: |
+          earthly-ci --no-output --push ./yarn-project+export-blob-sink --DIST_TAG=${{ env.GIT_COMMIT }} --ARCH=x86_64
+
+  build-blob-sink-arm:
+    needs: [configure, build-aztec-arm]
+    runs-on: ${{ needs.configure.outputs.username }}-arm
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: "${{ env.GIT_COMMIT }}"
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: build-aztec-blob-sink-arm64
+          dockerhub_password: "${{ env.DOCKERHUB_PASSWORD }}"
+      - name: Build & push aztec blob sink image arm
+        run: |
+          earthly-ci --no-output --push ./yarn-project+export-blob-sink --DIST_TAG=${{ env.GIT_COMMIT }} --ARCH=arm64
+
   publish-manifests:
     needs:
       - configure

aztec-up/bin/aztec-install

@@ -107,6 +107,7 @@ if [ -z "${SKIP_PULL:-}" ]; then
   pull_container aztec-nargo
   pull_container aztec
   pull_container cli-wallet
+  pull_container blob-sink
 fi
 
 # Download the Docker Compose file. Used by aztec.

aztec-up/bin/docker-compose.sandbox.yml

@@ -15,6 +15,13 @@ services:
       FORK_BLOCK_NUMBER:
       ANVIL_PORT: ${ANVIL_PORT:-8545}
 
+  # TODO: add a readiness probe to this
+  # TODO: delete all of these extra images and just have one, then symlink them???
+  blob-sink:
+    image: aztecprotocol/blob-sink
+    ports:
+      - "${BLOB_SINK_PORT:-5052}:${BLOB_SINK_PORT:-5052}"
+
   aztec:
     image: "aztecprotocol/aztec"
     ports:
@@ -29,11 +36,13 @@ services:
       SEQ_TX_POLLING_INTERVAL_MS: 50
       WS_BLOCK_CHECK_INTERVAL_MS: 50
       ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500
+      SEQ_BLOB_SINK_URL: http://blob-sink:${BLOB_SINK_PORT:-5052}
       PXE_PORT: ${PXE_PORT:-8080}
       PORT: ${AZTEC_NODE_PORT:-8080}
       TEST_ACCOUNTS: ${TEST_ACCOUNTS:-true}
     volumes:
       - ./log:/usr/src/yarn-project/aztec/log:rw
     depends_on:
       - ethereum
+      - blob-sink
     command: "start --sandbox"

boxes/docker-compose.yml

@@ -25,6 +25,7 @@ services:
       SEQ_TX_POLLING_INTERVAL_MS: 50
       WS_BLOCK_CHECK_INTERVAL_MS: 50
       ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500
+      SEQ_BLOB_SINK_URL: http://blob-sink:5052
     healthcheck:
       test: ["CMD", "curl", "-fSs", "http://127.0.0.1:8080/status"]
       interval: 3s
@@ -33,6 +34,13 @@ services:
     depends_on:
       - ethereum
 
+  blob-sink:
+    image: aztecprotocol/blob-sink:${AZTEC_DOCKER_TAG:-latest}
+    environment:
+      PORT: 5052
+      DEBUG: "aztec:*"
+      DEBUG_COLORS: "true"
+
   boxes:
     image: aztecprotocol/build:2.0
     tty: true

build_manifest.yml

@@ -242,6 +242,13 @@ cli-wallet:
     - yarn-project
   multiarch: buildx
 
+blob-sink:
+  buildDir: yarn-project
+  projectDir: yarn-project/blob-sink
+  dependencies:
+    - yarn-project
+  multiarch: buildx
+
 # Builds all the boxes. They are then independently tested in the container.
 boxes:
   buildDir: boxes

scripts/run_native_testnet.sh

@@ -128,7 +128,8 @@ BASE_CMD="INTERLEAVED=$INTERLEAVED ./yarn-project/end-to-end/scripts/native_netw
         \"./validators.sh $NUM_VALIDATORS\" \
         $PROVER_SCRIPT \
         ./pxe.sh \
-        ./transaction-bot.sh"
+        ./transaction-bot.sh \
+        ./blob-sink.sh"
 
 # Execute the command
 eval $BASE_CMD

yarn-project/Earthfile

@@ -22,6 +22,24 @@ export-cli-wallet:
     ARG ARCH
     SAVE IMAGE --push aztecprotocol/cli-wallet:${DIST_TAG}${ARCH:+-$ARCH}
 
+blob-sink-build:
+    FROM +bootstrap
+    RUN yarn workspaces focus @aztec/blob-sink --production && yarn cache clean
+    SAVE ARTIFACT /usr/src /usr/src
+
+blob-sink:
+    FROM ubuntu:noble
+    RUN apt update && apt install nodejs curl -y && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+    COPY +blob-sink-build/usr/src /usr/src
+    ENTRYPOINT ["node", "/usr/src/yarn-project/blob-sink/dest/run.js"]
+
+# TODO(ci3) move to bootstrap.sh
+export-blob-sink:
+    FROM +blob-sink
+    ARG DIST_TAG="latest"
+    ARG ARCH
+    SAVE IMAGE --push aztecprotocol/blob-sink:${DIST_TAG}${ARCH:+-$ARCH}
+
 anvil:
     FROM ../build-images+from-registry
     SAVE ARTIFACT /opt/foundry/bin/anvil

yarn-project/blob-sink/Dockerfile

@@ -0,0 +1,7 @@
+## TODO THIS
+
+FROM aztecprotocol/yarn-project AS yarn-project
+
+RUN apt update
+
+ENTRYPOINT ["node", "/usr/src/yarn-project/blob-sink/dest/bin/run.js"]

yarn-project/blob-sink/package.json

@@ -41,6 +41,77 @@
     "extensionsToTreatAsEsm": [
       ".ts"
     ],
+    "scripts": {
+      "build": "yarn clean && tsc -b",
+      "start": "node ./dest/bin/run.js",
+      "build:dev": "tsc -b --watch",
+      "clean": "rm -rf ./dest .tsbuildinfo",
+      "formatting": "run -T prettier --check ./src && run -T eslint ./src",
+      "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
+      "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests"
+    },
+    "jest": {
+      "moduleNameMapper": {
+        "^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
+      },
+      "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$",
+      "rootDir": "./src",
+      "transform": {
+        "^.+\\.tsx?$": [
+          "@swc/jest",
+          {
+            "jsc": {
+              "parser": {
+                "syntax": "typescript",
+                "decorators": true
+              },
+              "transform": {
+                "decoratorVersion": "2022-03"
+              }
+            }
+          }
+        ]
+      },
+      "extensionsToTreatAsEsm": [
+        ".ts"
+      ],
+      "reporters": [
+        [
+          "default",
+          {
+            "summaryThreshold": 9999
+          }
+        ]
+      ]
+    },
+    "dependencies": {
+      "@aztec/circuit-types": "workspace:^",
+      "@aztec/foundation": "workspace:^",
+      "@aztec/kv-store": "workspace:*",
+      "@aztec/telemetry-client": "workspace:*",
+      "express": "^4.21.1",
+      "source-map-support": "^0.5.21",
+      "tslib": "^2.4.0",
+      "zod": "^3.23.8"
+    },
+    "devDependencies": {
+      "@jest/globals": "^29.5.0",
+      "@types/jest": "^29.5.0",
+      "@types/memdown": "^3.0.0",
+      "@types/node": "^18.7.23",
+      "@types/source-map-support": "^0.5.10",
+      "@types/supertest": "^6.0.2",
+      "jest": "^29.5.0",
+      "jest-mock-extended": "^3.0.3",
+      "supertest": "^7.0.0",
+      "ts-node": "^10.9.1",
+      "typescript": "^5.0.4"
+    },
+    "files": [
+      "dest",
+      "src",
+      "!*.test.*"
+    ],
     "reporters": [
       "default"
     ],

yarn-project/blob-sink/src/config.ts

@@ -1,7 +1,30 @@
-import { type DataStoreConfig } from '@aztec/kv-store/config';
+import { type ConfigMappingsType, getConfigFromMappings } from '@aztec/foundation/config';
+import { type DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config';
 
-export interface BlobSinkConfig {
+export type BlobSinkConfig = {
   port?: number;
-  dataStoreConfig?: DataStoreConfig;
-  otelMetricsCollectorUrl?: string;
+  // otelMetricsCollectorUrl?: string;
+} & DataStoreConfig;
+
+export const blobSinkConfigMappings: ConfigMappingsType<BlobSinkConfig> = {
+  port: {
+    env: 'BLOB_SINK_PORT',
+    description: 'The port to run the blob sink server on',
+  },
+  ...dataConfigMappings,
+
+  // TODO: bring otel endpoints back
+  // otelMetricsCollectorUrl: {
+  //   env: 'OTEL_METRICS_COLLECTOR_URL',
+  //   description: 'The URL of the OTLP metrics collector',
+  // },
+};
+
+/**
+ * Returns the blob sink configuration from the environment variables.
+ * Note: If an environment variable is not set, the default value is used.
+ * @returns The blob sink configuration.
+ */
+export function getBlobSinkConfigFromEnv(): BlobSinkConfig {
+  return getConfigFromMappings<BlobSinkConfig>(blobSinkConfigMappings);
 }

yarn-project/blob-sink/src/factory.ts

@@ -8,10 +8,10 @@ import { BlobSinkServer } from './server.js';
 // If data store settings are provided, the store is created and returned.
 // Otherwise, undefined is returned and an in memory store will be used.
 async function getDataStoreConfig(config?: BlobSinkConfig): Promise<AztecKVStore | undefined> {
-  if (!config?.dataStoreConfig) {
+  if (!config?.dataDirectory) {
     return undefined;
   }
-  return await createStore('blob-sink', config.dataStoreConfig);
+  return await createStore('blob-sink', config);
 }
 
 /**

yarn-project/blob-sink/src/run.ts

@@ -0,0 +1,25 @@
+// Run a standalone blob sink server
+import { createLogger } from '@aztec/foundation/log';
+
+import { getBlobSinkConfigFromEnv } from './config.js';
+import { BlobSinkServer } from './server.js';
+
+const logger = createLogger('aztec:blob-sink');
+
+async function main() {
+  const config = getBlobSinkConfigFromEnv();
+  const blobSinkServer = new BlobSinkServer(config);
+
+  await blobSinkServer.start();
+
+  const stop = async () => {
+    logger.debug('Stopping Blob Sink...');
+    await blobSinkServer.stop();
+    logger.info('Node stopped');
+    process.exit(0);
+  };
+  process.on('SIGTERM', stop);
+  process.on('SIGINT', stop);
+}
+
+void main();

yarn-project/blob-sink/src/server.ts

@@ -31,7 +31,11 @@ export class BlobSinkServer {
   private metrics: BlobSinkMetrics;
   private log: Logger = createLogger('aztec:blob-sink');
 
-  constructor(config?: BlobSinkConfig, store?: AztecKVStore, telemetry: TelemetryClient = new NoopTelemetryClient()) {
+  constructor(
+    config?: Partial<BlobSinkConfig>,
+    store?: AztecKVStore,
+    telemetry: TelemetryClient = new NoopTelemetryClient(),
+  ) {
     this.port = config?.port ?? 5052; // 5052 is beacon chain default http port
     this.app = express();
 

yarn-project/end-to-end/scripts/docker-compose.yml

@@ -21,9 +21,17 @@ services:
       SEQ_TX_POLLING_INTERVAL_MS: 500
       WS_BLOCK_CHECK_INTERVAL_MS: 500
       ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500
+      SEQ_BLOB_SINK_URL: http://blob-sink:${BLOB_SINK_PORT:-5052}
       ENABLE_GAS: ${ENABLE_GAS:-}
       HARDWARE_CONCURRENCY: ${HARDWARE_CONCURRENCY:-}
 
+  blob-sink:
+    image: aztecprotocol/blob-sink
+    environment:
+      PORT: ${BLOB_SINK_PORT:-5052}
+      DEBUG: 'aztec:*'
+      DEBUG_COLORS: 1
+
   end-to-end:
     image: aztecprotocol/build:2.0
     volumes:

yarn-project/end-to-end/scripts/native-network/blob-sink.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -eu
+
+REPO=$(git rev-parse --show-toplevel)
+
+# Starts the Blob Sink
+export PORT=${BLOB_SINK_PORT:-5052}
+export DEBUG=${DEBUG:-"aztec:*"}
+export DEBUG_COLORS=${DEBUG_COLORS:-1}
+
+node --no-warnings "$REPO"/yarn-project/blob-sink/dest/run.js

yarn-project/end-to-end/scripts/native-network/prover-node.sh

@@ -40,6 +40,7 @@ export PROVER_AGENT_ENABLED="true"
 export PROVER_PUBLISHER_PRIVATE_KEY=${PROVER_PUBLISHER_PRIVATE_KEY:-"0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"}
 export PROVER_COORDINATION_NODE_URL="http://127.0.0.1:8080"
 export AZTEC_NODE_URL="http://127.0.0.1:8080"
+export PROVER_BLOB_SINK_URL="http://127.0.0.1:${BLOB_SINK_PORT:-5052}"
 export OTEL_RESOURCE_ATTRIBUTES="service.name=prover-node-${PORT}"
 export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-}"
 export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-}"

yarn-project/end-to-end/scripts/native-network/validator.sh

@@ -68,6 +68,7 @@ export P2P_TCP_ANNOUNCE_ADDR="127.0.0.1:$P2P_PORT"
 export P2P_UDP_ANNOUNCE_ADDR="127.0.0.1:$P2P_PORT"
 export P2P_TCP_LISTEN_ADDR="0.0.0.0:$P2P_PORT"
 export P2P_UDP_LISTEN_ADDR="0.0.0.0:$P2P_PORT"
+export SEQ_BLOB_SINK_URL="http://127.0.0.1:${BLOB_SINK_PORT:-5052}"
 export OTEL_RESOURCE_ATTRIBUTES="service.name=validator-node-${PORT}"
 export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-}"
 export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-}"

yarn-project/end-to-end/src/fixtures/snapshot_manager.ts

@@ -451,10 +451,8 @@ async function setupFromState(statePath: string, logger: Logger): Promise<Subsys
 
   const blobSink = await createBlobSinkServer({
     port: blobSinkPort,
-    dataStoreConfig: {
-      dataDirectory: statePath,
-      dataStoreMapSizeKB: aztecNodeConfig.dataStoreMapSizeKB,
-    },
+    dataDirectory: statePath,
+    dataStoreMapSizeKB: aztecNodeConfig.dataStoreMapSizeKB,
   });
   await blobSink.start();
 

yarn-project/end-to-end/src/fixtures/utils.ts

@@ -389,7 +389,11 @@ export async function setup(
 
   // Blob sink service - blobs get posted here and served from here
   const blobSinkPort = await getPort();
-  const blobSink = await createBlobSinkServer({ port: blobSinkPort });
+  const blobSink = await createBlobSinkServer({
+    port: blobSinkPort,
+    dataDirectory: undefined,
+    dataStoreMapSizeKB: 1, // will be unused, set as default
+  });
   config.blobSinkUrl = `http://127.0.0.1:${blobSinkPort}`;
 
   const deployL1ContractsValues =

yarn-project/foundation/src/config/env_var.ts

@@ -36,6 +36,7 @@ export type EnvVar =
   | 'BOT_TX_MINED_WAIT_SECONDS'
   | 'BOT_MAX_CONSECUTIVE_ERRORS'
   | 'BOT_STOP_WHEN_UNHEALTHY'
+  | 'BLOB_SINK_PORT'
   | 'COINBASE'
   | 'DATA_DIRECTORY'
   | 'DATA_STORE_MAP_SIZE_KB'