diff --git a/barretenberg/cpp/srs_db/download_ignition.sh b/barretenberg/cpp/srs_db/download_ignition.sh index 478f442f9af..87c86a8f88b 100755 --- a/barretenberg/cpp/srs_db/download_ignition.sh +++ b/barretenberg/cpp/srs_db/download_ignition.sh @@ -20,6 +20,9 @@ cd ignition mkdir -p monomial cd monomial NUM=${1:-19} +RANGE_START=${2:-} +RANGE_END=${3:-} +APPEND=${4:-"false"} if command -v sha256sum > /dev/null; then SHASUM=sha256sum @@ -33,12 +36,26 @@ checksum() { } download() { - curl https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat > transcript${1}.dat + # Initialize an empty variable for the Range header + RANGE_HEADER="" + + # If both RANGE_START and RANGE_END are set, add them to the Range header + if [ -n "$RANGE_START" ] && [ -n "$RANGE_END" ]; then + RANGE_HEADER="-H Range:bytes=$RANGE_START-$RANGE_END" + fi + + # Download the file + if [ "$APPEND" = "true" ]; then + curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat >> transcript${1}.dat + else + curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat > transcript${1}.dat + fi + } for TRANSCRIPT in $(seq 0 $NUM); do NUM=$(printf %02d $TRANSCRIPT) - if [ -f checksums ]; then + if [ -f checksums ] && [ -z "$RANGE_START" ] && [ -z "$RANGE_END" ] ; then checksum $NUM && continue download $NUM checksum $NUM || exit 1 diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index 9eb4dcc4ffe..1794668b6e0 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -19,4 +19,13 @@ RUN yarn tsc -b FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base COPY --from=builder /usr/src/yarn-project /usr/src/yarn-project + +# Download minimal CRS requirements +COPY --from=builder /usr/src/barretenberg/cpp/srs_db/download_ignition.sh /usr/src/yarn-project/circuits.js/resources/download_ignition.sh +WORKDIR /usr/src/yarn-project/circuits.js/resources +RUN ./download_ignition.sh 0 28 3200027 +RUN ./download_ignition.sh 0 322560092 322560219 true + +WORKDIR /usr/src/yarn-project + ENTRYPOINT ["yarn"] \ No newline at end of file diff --git a/yarn-project/circuits.js/.gitignore b/yarn-project/circuits.js/.gitignore new file mode 100644 index 00000000000..76fc89283e7 --- /dev/null +++ b/yarn-project/circuits.js/.gitignore @@ -0,0 +1 @@ +resources/ignition/ \ No newline at end of file diff --git a/yarn-project/circuits.js/resources/download_ignition.sh b/yarn-project/circuits.js/resources/download_ignition.sh new file mode 100755 index 00000000000..87c86a8f88b --- /dev/null +++ b/yarn-project/circuits.js/resources/download_ignition.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# Downloads the ignition trusted setup transcripts. +# +# See here for details of the contents of the transcript.dat files: +# https://github.com/AztecProtocol/ignition-verification/blob/master/Transcript_spec.md +# +# To download all transcripts. +# ./download_ignition.sh +# +# To download a range of transcripts, e.g. 0, 1 and 2. +# ./download_ignition.sh 2 +# +# If a checksums file is available, it will be used to validate if a download is required +# and also check the validity of the downloaded transcripts. If not the script downloads +# whatever is requested but does not check the validity of the downloads. +set -eu + +mkdir -p ignition +cd ignition +mkdir -p monomial +cd monomial +NUM=${1:-19} +RANGE_START=${2:-} +RANGE_END=${3:-} +APPEND=${4:-"false"} + +if command -v sha256sum > /dev/null; then + SHASUM=sha256sum +else + SHASUM="shasum -a 256" +fi + +checksum() { + grep transcript${1}.dat checksums | $SHASUM -c + return $? +} + +download() { + # Initialize an empty variable for the Range header + RANGE_HEADER="" + + # If both RANGE_START and RANGE_END are set, add them to the Range header + if [ -n "$RANGE_START" ] && [ -n "$RANGE_END" ]; then + RANGE_HEADER="-H Range:bytes=$RANGE_START-$RANGE_END" + fi + + # Download the file + if [ "$APPEND" = "true" ]; then + curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat >> transcript${1}.dat + else + curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat > transcript${1}.dat + fi + +} + +for TRANSCRIPT in $(seq 0 $NUM); do + NUM=$(printf %02d $TRANSCRIPT) + if [ -f checksums ] && [ -z "$RANGE_START" ] && [ -z "$RANGE_END" ] ; then + checksum $NUM && continue + download $NUM + checksum $NUM || exit 1 + else + download $NUM + fi +done diff --git a/yarn-project/circuits.js/src/crs/index.ts b/yarn-project/circuits.js/src/crs/index.ts index f91cb772686..9a9530af9fb 100644 --- a/yarn-project/circuits.js/src/crs/index.ts +++ b/yarn-project/circuits.js/src/crs/index.ts @@ -1,9 +1,10 @@ +import { createDebugLogger } from '@aztec/foundation/log'; import { fileURLToPath } from '@aztec/foundation/url'; import isNode from 'detect-node'; import { existsSync } from 'fs'; import { open } from 'fs/promises'; -import { dirname } from 'path'; +import { dirname, join } from 'path'; /** * Downloader for CRS from the web or local. @@ -26,7 +27,6 @@ export class NetCrs { // We need numPoints number of g1 points. const g1Start = 28; const g1End = g1Start + this.numPoints * 64 - 1; - // Download required range of data. const response = await fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/monomial/transcript00.dat', { headers: { @@ -43,7 +43,8 @@ export class NetCrs { * Download the G2 points data. */ async downloadG2Data() { - const g2Start = 28 + 5040001 * 64; + const g2Start = 28 + 5_040_001 * 64; // = 322_560_092 + const g2End = g2Start + 128 - 1; const response2 = await fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/monomial/transcript00.dat', { @@ -85,6 +86,7 @@ export class FileCrs { */ public readonly numPoints: number, private path: string, + private readonly offsetStart = true, ) {} /** @@ -92,10 +94,10 @@ export class FileCrs { */ async init() { // We need numPoints number of g1 points. - const g1Start = 28; + const g1Start = this.offsetStart ? 28 : 0; const g1Length = this.numPoints * 64; - const g2Start = 28 + 5040001 * 64; + const g2Start = 28 + 5_040_001 * 64; // = 322_560_092 const g2Length = 128; // Lazily seek our data const fileHandle = await open(this.path, 'r'); @@ -104,7 +106,7 @@ export class FileCrs { await fileHandle.read(this.data, 0, g1Length, g1Start); this.g2Data = Buffer.alloc(g2Length); - await fileHandle.read(this.g2Data, 0, g2Length, g2Start); + await fileHandle.read(this.g2Data, 0, g2Length, this.offsetStart ? g2Start : g1Start + g1Length); } finally { await fileHandle.close(); } @@ -132,21 +134,40 @@ export class FileCrs { */ export class Crs { private crs: FileCrs | NetCrs; + private logger = createDebugLogger('circuits:crs'); + /** + * The path to our SRS object, assuming that we are in the aztec3-packages folder structure. + */ + private devPath = '/../../../../barretenberg/cpp/srs_db/ignition/monomial/transcript00.dat'; + /** + * The path of our SRS object, if we downloaded on init. + */ + private localPath = `/../../resources/ignition/monomial/transcript00.dat`; constructor( /** * The number of circuit gates. */ public readonly numPoints: number, + + /** + * Option to save downloaded SRS on file. + */ + private readonly saveOnFile = true, ) { if (isNode) { - /** - * The path to our SRS object, assuming that we are in the aztec3-packages folder structure. - */ - const SRS_DEV_PATH = - dirname(fileURLToPath(import.meta.url)) + - '/../../../../barretenberg/cpp/srs_db/ignition/monomial/transcript00.dat'; - this.crs = existsSync(SRS_DEV_PATH) ? new FileCrs(numPoints, SRS_DEV_PATH) : new NetCrs(numPoints); + const devPath = join(fileURLToPath(import.meta.url), this.devPath); + const localPath = join(dirname(fileURLToPath(import.meta.url)), this.localPath); + const existsDev = existsSync(devPath); + const existsLocal = existsSync(localPath); + + if (existsDev) { + this.crs = new FileCrs(numPoints, devPath); + } else if (existsLocal) { + this.crs = new FileCrs(numPoints, localPath, false); + } else { + this.crs = new NetCrs(numPoints); + } } else { this.crs = new NetCrs(numPoints); } @@ -157,6 +178,29 @@ export class Crs { */ async init() { await this.crs.init(); + if (isNode) { + const localPath = dirname(fileURLToPath(import.meta.url)) + this.localPath; + // save downloaded CRS on file + if (this.saveOnFile && !existsSync(localPath)) { + const fileHandle = await open(localPath, 'w'); + const g1Data = Buffer.from(this.crs.getG1Data()); + try { + await fileHandle.write(g1Data); + } catch (err: any) { + this.logger.warn('Failed to save CRS data: ', err.message); + } + + const g2Data = Buffer.from(this.crs.getG2Data()); + try { + await fileHandle.write(g2Data, 0, g2Data.length, g1Data.length); + // appendFileSync(localPath, Buffer.from(g2Data)); + } catch (err: any) { + this.logger.warn('Failed to append G2 data: ', err.message); + } finally { + await fileHandle.close(); + } + } + } } /** diff --git a/yarn-project/deploy_npm.sh b/yarn-project/deploy_npm.sh index 8b9a54c610c..a80cbc5ad2b 100755 --- a/yarn-project/deploy_npm.sh +++ b/yarn-project/deploy_npm.sh @@ -74,4 +74,3 @@ deploy_package world-state deploy_package key-store deploy_package aztec-node deploy_package aztec-sandbox - diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index 6c0e471ea43..e9c9b521106 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -44,7 +44,8 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/circuits-wasm-linux-clang as c FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/l1-contracts as contracts FROM node:18-alpine -RUN apk update && apk add --no-cache bash jq +RUN apk update && apk add --no-cache bash jq curl + WORKDIR /usr/src/yarn-project # The dockerignore file ensures the context only contains package.json and tsconfig.json files. @@ -69,6 +70,9 @@ RUN yarn prepare:check # Bring in circuits wasms. COPY --from=circuits /usr/src/circuits/cpp/build-wasm/bin /usr/src/circuits/cpp/build-wasm/bin +# Copy ignition download script. +COPY --from=circuits /usr/src/barretenberg/cpp/srs_db/download_ignition.sh /usr/src/barretenberg/cpp/srs_db/download_ignition.sh + # Generate L1 contract TypeScript artifacts. COPY --from=contracts /usr/src/l1-contracts/out /usr/src/l1-contracts/out RUN cd l1-artifacts && ./scripts/generate-artifacts.sh