diff --git a/.github/workflows/publish-aztec-packages.yml b/.github/workflows/publish-aztec-packages.yml index 0f1549677aa..ce0b12dd85e 100644 --- a/.github/workflows/publish-aztec-packages.yml +++ b/.github/workflows/publish-aztec-packages.yml @@ -183,6 +183,36 @@ jobs: run: | earthly-ci --no-output --push ./yarn-project+export-cli-wallet --DIST_TAG=${{ env.GIT_COMMIT }} --ARCH=arm64 + build-blob-sink-x86: + needs: [configure, build-aztec-x86] + runs-on: ${{ needs.configure.outputs.username }}-x86 + steps: + - uses: actions/checkout@v4 + with: + ref: "${{ env.GIT_COMMIT }}" + - uses: ./.github/ci-setup-action + with: + concurrency_key: build-aztec-blob-sink-x86_64 + dockerhub_password: "${{ env.DOCKERHUB_PASSWORD }}" + - name: Build & push aztec blob sink image x86-64 + run: | + earthly-ci --no-output --push ./yarn-project+export-blob-sink --DIST_TAG=${{ env.GIT_COMMIT }} --ARCH=x86_64 + + build-blob-sink-arm: + needs: [configure, build-aztec-arm] + runs-on: ${{ needs.configure.outputs.username }}-arm + steps: + - uses: actions/checkout@v4 + with: + ref: "${{ env.GIT_COMMIT }}" + - uses: ./.github/ci-setup-action + with: + concurrency_key: build-aztec-blob-sink-arm64 + dockerhub_password: "${{ env.DOCKERHUB_PASSWORD }}" + - name: Build & push aztec blob sink image arm + run: | + earthly-ci --no-output --push ./yarn-project+export-blob-sink --DIST_TAG=${{ env.GIT_COMMIT }} --ARCH=arm64 + publish-manifests: needs: - configure diff --git a/aztec-up/bin/aztec-install b/aztec-up/bin/aztec-install index 470664f7040..51c1bcb41ab 100755 --- a/aztec-up/bin/aztec-install +++ b/aztec-up/bin/aztec-install @@ -107,6 +107,7 @@ if [ -z "${SKIP_PULL:-}" ]; then pull_container aztec-nargo pull_container aztec pull_container cli-wallet + pull_container blob-sink fi # Download the Docker Compose file. Used by aztec. diff --git a/aztec-up/bin/docker-compose.sandbox.yml b/aztec-up/bin/docker-compose.sandbox.yml index 999aa567685..184f85fb378 100644 --- a/aztec-up/bin/docker-compose.sandbox.yml +++ b/aztec-up/bin/docker-compose.sandbox.yml @@ -15,6 +15,13 @@ services: FORK_BLOCK_NUMBER: ANVIL_PORT: ${ANVIL_PORT:-8545} + # TODO: add a readiness probe to this + # TODO: delete all of these extra images and just have one, then symlink them??? + blob-sink: + image: aztecprotocol/blob-sink + ports: + - "${BLOB_SINK_PORT:-5052}:${BLOB_SINK_PORT:-5052}" + aztec: image: "aztecprotocol/aztec" ports: @@ -29,6 +36,7 @@ services: SEQ_TX_POLLING_INTERVAL_MS: 50 WS_BLOCK_CHECK_INTERVAL_MS: 50 ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 + SEQ_BLOB_SINK_URL: http://blob-sink:${BLOB_SINK_PORT:-5052} PXE_PORT: ${PXE_PORT:-8080} PORT: ${AZTEC_NODE_PORT:-8080} TEST_ACCOUNTS: ${TEST_ACCOUNTS:-true} @@ -36,4 +44,5 @@ services: - ./log:/usr/src/yarn-project/aztec/log:rw depends_on: - ethereum + - blob-sink command: "start --sandbox" diff --git a/boxes/docker-compose.yml b/boxes/docker-compose.yml index 73791dfdc9b..1f56e993f7c 100644 --- a/boxes/docker-compose.yml +++ b/boxes/docker-compose.yml @@ -25,6 +25,7 @@ services: SEQ_TX_POLLING_INTERVAL_MS: 50 WS_BLOCK_CHECK_INTERVAL_MS: 50 ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 + SEQ_BLOB_SINK_URL: http://blob-sink:5052 healthcheck: test: ["CMD", "curl", "-fSs", "http://127.0.0.1:8080/status"] interval: 3s @@ -33,6 +34,13 @@ services: depends_on: - ethereum + blob-sink: + image: aztecprotocol/blob-sink:${AZTEC_DOCKER_TAG:-latest} + environment: + PORT: 5052 + DEBUG: "aztec:*" + DEBUG_COLORS: "true" + boxes: image: aztecprotocol/build:2.0 tty: true diff --git a/build_manifest.yml b/build_manifest.yml index 54817416bad..b8d31c7d2be 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -242,6 +242,13 @@ cli-wallet: - yarn-project multiarch: buildx +blob-sink: + buildDir: yarn-project + projectDir: yarn-project/blob-sink + dependencies: + - yarn-project + multiarch: buildx + # Builds all the boxes. They are then independently tested in the container. boxes: buildDir: boxes diff --git a/scripts/run_native_testnet.sh b/scripts/run_native_testnet.sh index 4cf6d83900a..2e6bc2c433b 100755 --- a/scripts/run_native_testnet.sh +++ b/scripts/run_native_testnet.sh @@ -128,7 +128,8 @@ BASE_CMD="INTERLEAVED=$INTERLEAVED ./yarn-project/end-to-end/scripts/native_netw \"./validators.sh $NUM_VALIDATORS\" \ $PROVER_SCRIPT \ ./pxe.sh \ - ./transaction-bot.sh" + ./transaction-bot.sh \ + ./blob-sink.sh" # Execute the command eval $BASE_CMD diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index 0cc2899657f..9e2d9d43b4c 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -22,6 +22,24 @@ export-cli-wallet: ARG ARCH SAVE IMAGE --push aztecprotocol/cli-wallet:${DIST_TAG}${ARCH:+-$ARCH} +blob-sink-build: + FROM +bootstrap + RUN yarn workspaces focus @aztec/blob-sink --production && yarn cache clean + SAVE ARTIFACT /usr/src /usr/src + +blob-sink: + FROM ubuntu:noble + RUN apt update && apt install nodejs curl -y && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + COPY +blob-sink-build/usr/src /usr/src + ENTRYPOINT ["node", "/usr/src/yarn-project/blob-sink/dest/run.js"] + +# TODO(ci3) move to bootstrap.sh +export-blob-sink: + FROM +blob-sink + ARG DIST_TAG="latest" + ARG ARCH + SAVE IMAGE --push aztecprotocol/blob-sink:${DIST_TAG}${ARCH:+-$ARCH} + anvil: FROM ../build-images+from-registry SAVE ARTIFACT /opt/foundry/bin/anvil diff --git a/yarn-project/blob-sink/Dockerfile b/yarn-project/blob-sink/Dockerfile new file mode 100644 index 00000000000..610ea2b398f --- /dev/null +++ b/yarn-project/blob-sink/Dockerfile @@ -0,0 +1,7 @@ +## TODO THIS + +FROM aztecprotocol/yarn-project AS yarn-project + +RUN apt update + +ENTRYPOINT ["node", "/usr/src/yarn-project/blob-sink/dest/bin/run.js"] diff --git a/yarn-project/blob-sink/package.json b/yarn-project/blob-sink/package.json index 7090bf99527..cd42c5a3874 100644 --- a/yarn-project/blob-sink/package.json +++ b/yarn-project/blob-sink/package.json @@ -41,6 +41,77 @@ "extensionsToTreatAsEsm": [ ".ts" ], + "scripts": { + "build": "yarn clean && tsc -b", + "start": "node ./dest/bin/run.js", + "build:dev": "tsc -b --watch", + "clean": "rm -rf ./dest .tsbuildinfo", + "formatting": "run -T prettier --check ./src && run -T eslint ./src", + "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", + "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests" + }, + "jest": { + "moduleNameMapper": { + "^(\\.{1,2}/.*)\\.[cm]?js$": "$1" + }, + "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$", + "rootDir": "./src", + "transform": { + "^.+\\.tsx?$": [ + "@swc/jest", + { + "jsc": { + "parser": { + "syntax": "typescript", + "decorators": true + }, + "transform": { + "decoratorVersion": "2022-03" + } + } + } + ] + }, + "extensionsToTreatAsEsm": [ + ".ts" + ], + "reporters": [ + [ + "default", + { + "summaryThreshold": 9999 + } + ] + ] + }, + "dependencies": { + "@aztec/circuit-types": "workspace:^", + "@aztec/foundation": "workspace:^", + "@aztec/kv-store": "workspace:*", + "@aztec/telemetry-client": "workspace:*", + "express": "^4.21.1", + "source-map-support": "^0.5.21", + "tslib": "^2.4.0", + "zod": "^3.23.8" + }, + "devDependencies": { + "@jest/globals": "^29.5.0", + "@types/jest": "^29.5.0", + "@types/memdown": "^3.0.0", + "@types/node": "^18.7.23", + "@types/source-map-support": "^0.5.10", + "@types/supertest": "^6.0.2", + "jest": "^29.5.0", + "jest-mock-extended": "^3.0.3", + "supertest": "^7.0.0", + "ts-node": "^10.9.1", + "typescript": "^5.0.4" + }, + "files": [ + "dest", + "src", + "!*.test.*" + ], "reporters": [ "default" ], diff --git a/yarn-project/blob-sink/src/config.ts b/yarn-project/blob-sink/src/config.ts index e18311f9f1d..0dfb0e57c12 100644 --- a/yarn-project/blob-sink/src/config.ts +++ b/yarn-project/blob-sink/src/config.ts @@ -1,7 +1,30 @@ -import { type DataStoreConfig } from '@aztec/kv-store/config'; +import { type ConfigMappingsType, getConfigFromMappings } from '@aztec/foundation/config'; +import { type DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config'; -export interface BlobSinkConfig { +export type BlobSinkConfig = { port?: number; - dataStoreConfig?: DataStoreConfig; - otelMetricsCollectorUrl?: string; + // otelMetricsCollectorUrl?: string; +} & DataStoreConfig; + +export const blobSinkConfigMappings: ConfigMappingsType = { + port: { + env: 'BLOB_SINK_PORT', + description: 'The port to run the blob sink server on', + }, + ...dataConfigMappings, + + // TODO: bring otel endpoints back + // otelMetricsCollectorUrl: { + // env: 'OTEL_METRICS_COLLECTOR_URL', + // description: 'The URL of the OTLP metrics collector', + // }, +}; + +/** + * Returns the blob sink configuration from the environment variables. + * Note: If an environment variable is not set, the default value is used. + * @returns The blob sink configuration. + */ +export function getBlobSinkConfigFromEnv(): BlobSinkConfig { + return getConfigFromMappings(blobSinkConfigMappings); } diff --git a/yarn-project/blob-sink/src/factory.ts b/yarn-project/blob-sink/src/factory.ts index 43a0df8e6c3..b1c362779c2 100644 --- a/yarn-project/blob-sink/src/factory.ts +++ b/yarn-project/blob-sink/src/factory.ts @@ -8,10 +8,10 @@ import { BlobSinkServer } from './server.js'; // If data store settings are provided, the store is created and returned. // Otherwise, undefined is returned and an in memory store will be used. async function getDataStoreConfig(config?: BlobSinkConfig): Promise { - if (!config?.dataStoreConfig) { + if (!config?.dataDirectory) { return undefined; } - return await createStore('blob-sink', config.dataStoreConfig); + return await createStore('blob-sink', config); } /** diff --git a/yarn-project/blob-sink/src/run.ts b/yarn-project/blob-sink/src/run.ts new file mode 100644 index 00000000000..02c240691c9 --- /dev/null +++ b/yarn-project/blob-sink/src/run.ts @@ -0,0 +1,25 @@ +// Run a standalone blob sink server +import { createLogger } from '@aztec/foundation/log'; + +import { getBlobSinkConfigFromEnv } from './config.js'; +import { BlobSinkServer } from './server.js'; + +const logger = createLogger('aztec:blob-sink'); + +async function main() { + const config = getBlobSinkConfigFromEnv(); + const blobSinkServer = new BlobSinkServer(config); + + await blobSinkServer.start(); + + const stop = async () => { + logger.debug('Stopping Blob Sink...'); + await blobSinkServer.stop(); + logger.info('Node stopped'); + process.exit(0); + }; + process.on('SIGTERM', stop); + process.on('SIGINT', stop); +} + +void main(); diff --git a/yarn-project/blob-sink/src/server.ts b/yarn-project/blob-sink/src/server.ts index 167ead5de91..a861a982f73 100644 --- a/yarn-project/blob-sink/src/server.ts +++ b/yarn-project/blob-sink/src/server.ts @@ -31,7 +31,11 @@ export class BlobSinkServer { private metrics: BlobSinkMetrics; private log: Logger = createLogger('aztec:blob-sink'); - constructor(config?: BlobSinkConfig, store?: AztecKVStore, telemetry: TelemetryClient = new NoopTelemetryClient()) { + constructor( + config?: Partial, + store?: AztecKVStore, + telemetry: TelemetryClient = new NoopTelemetryClient(), + ) { this.port = config?.port ?? 5052; // 5052 is beacon chain default http port this.app = express(); diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 50e77c31515..5203ef37086 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -21,9 +21,17 @@ services: SEQ_TX_POLLING_INTERVAL_MS: 500 WS_BLOCK_CHECK_INTERVAL_MS: 500 ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 + SEQ_BLOB_SINK_URL: http://blob-sink:${BLOB_SINK_PORT:-5052} ENABLE_GAS: ${ENABLE_GAS:-} HARDWARE_CONCURRENCY: ${HARDWARE_CONCURRENCY:-} + blob-sink: + image: aztecprotocol/blob-sink + environment: + PORT: ${BLOB_SINK_PORT:-5052} + DEBUG: 'aztec:*' + DEBUG_COLORS: 1 + end-to-end: image: aztecprotocol/build:2.0 volumes: diff --git a/yarn-project/end-to-end/scripts/native-network/blob-sink.sh b/yarn-project/end-to-end/scripts/native-network/blob-sink.sh new file mode 100755 index 00000000000..b9ae6482003 --- /dev/null +++ b/yarn-project/end-to-end/scripts/native-network/blob-sink.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -eu + +REPO=$(git rev-parse --show-toplevel) + +# Starts the Blob Sink +export PORT=${BLOB_SINK_PORT:-5052} +export DEBUG=${DEBUG:-"aztec:*"} +export DEBUG_COLORS=${DEBUG_COLORS:-1} + +node --no-warnings "$REPO"/yarn-project/blob-sink/dest/run.js diff --git a/yarn-project/end-to-end/scripts/native-network/prover-node.sh b/yarn-project/end-to-end/scripts/native-network/prover-node.sh index 08a4c748855..a68c1f6da55 100755 --- a/yarn-project/end-to-end/scripts/native-network/prover-node.sh +++ b/yarn-project/end-to-end/scripts/native-network/prover-node.sh @@ -40,6 +40,7 @@ export PROVER_AGENT_ENABLED="true" export PROVER_PUBLISHER_PRIVATE_KEY=${PROVER_PUBLISHER_PRIVATE_KEY:-"0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"} export PROVER_COORDINATION_NODE_URL="http://127.0.0.1:8080" export AZTEC_NODE_URL="http://127.0.0.1:8080" +export PROVER_BLOB_SINK_URL="http://127.0.0.1:${BLOB_SINK_PORT:-5052}" export OTEL_RESOURCE_ATTRIBUTES="service.name=prover-node-${PORT}" export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-}" export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-}" diff --git a/yarn-project/end-to-end/scripts/native-network/validator.sh b/yarn-project/end-to-end/scripts/native-network/validator.sh index 580b60483b6..aa05a139a32 100755 --- a/yarn-project/end-to-end/scripts/native-network/validator.sh +++ b/yarn-project/end-to-end/scripts/native-network/validator.sh @@ -68,6 +68,7 @@ export P2P_TCP_ANNOUNCE_ADDR="127.0.0.1:$P2P_PORT" export P2P_UDP_ANNOUNCE_ADDR="127.0.0.1:$P2P_PORT" export P2P_TCP_LISTEN_ADDR="0.0.0.0:$P2P_PORT" export P2P_UDP_LISTEN_ADDR="0.0.0.0:$P2P_PORT" +export SEQ_BLOB_SINK_URL="http://127.0.0.1:${BLOB_SINK_PORT:-5052}" export OTEL_RESOURCE_ATTRIBUTES="service.name=validator-node-${PORT}" export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-}" export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-}" diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts index 836eb8c3c5b..00cc8a2cf34 100644 --- a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts +++ b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts @@ -451,10 +451,8 @@ async function setupFromState(statePath: string, logger: Logger): Promise