diff --git a/.circleci/config.yml b/.circleci/config.yml index 3155dcb7586..55879b23958 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -435,67 +435,52 @@ jobs: command: build noir-projects aztec_manifest_key: noir-projects - boxes-files: - machine: - image: default - resource_class: medium + yarn-project-pre-join: + docker: + - image: cimg/base:2023.09 + resource_class: small steps: - - *checkout - - *setup_env - run: - name: "Build" - command: build boxes-files - aztec_manifest_key: boxes-files + name: "Noop" + command: echo Noop - yarn-project-base: - machine: - image: default - resource_class: large + yarn-project-x86_64: + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Build" - command: build yarn-project-base | add_timestamps - aztec_manifest_key: yarn-project-base + command: cond_spot_run_build yarn-project 64 + aztec_manifest_key: yarn-project - yarn-project: - machine: - image: default - resource_class: large + yarn-project-arm64: + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: - name: Build - command: build yarn-project | add_timestamps + name: "Build" + command: cond_spot_run_build yarn-project 64 arm64 aztec_manifest_key: yarn-project - yarn-project-prod: + yarn-project-ecr-manifest: machine: image: default - resource_class: large - steps: - - *checkout - - *setup_env - - run: - name: Build - command: build yarn-project-prod | add_timestamps - aztec_manifest_key: yarn-project-prod - - yarn-project-formatting: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + resource_class: medium steps: - *checkout - *setup_env - run: - name: Check Formatting - command: cond_spot_run_container yarn-project 8 formatting | add_timestamps + name: "Create ECR manifest" + command: create_ecr_manifest yarn-project x86_64,arm64 aztec_manifest_key: yarn-project - yarn-project-tests: + yarn-project-test: docker: - image: aztecprotocol/alpine-build-image resource_class: small @@ -503,9 +488,9 @@ jobs: - *checkout - *setup_env - run: - name: Test - command: cond_spot_run_container yarn-project 64 test | add_timestamps - aztec_manifest_key: yarn-project + name: "Build and test" + command: cond_spot_run_build yarn-project-test 64 + aztec_manifest_key: yarn-project-test aztec-package: machine: @@ -985,7 +970,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml TEST=e2e_aztec_js_browser.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_aztec_js_browser.test.ts aztec_manifest_key: end-to-end e2e-card-game: @@ -1151,9 +1136,6 @@ jobs: steps: - *checkout - *setup_env - - run: - name: "Copy docs dockerignore" - command: cp docs/.dockerignore . - run: name: "Build docs" command: | @@ -1317,14 +1299,14 @@ defaults: &defaults event: fail branch_pattern: "master" -defaults_yarn_project: &defaults_yarn_project +defaults_yarn_project_pre_join: &defaults_yarn_project_pre_join requires: - - yarn-project + - yarn-project-pre-join <<: *defaults -defaults_yarn_project_prod: &defaults_yarn_project_prod +defaults_yarn_project: &defaults_yarn_project requires: - - yarn-project-prod + - yarn-project-ecr-manifest <<: *defaults defaults_deploy: &defaults_deploy @@ -1418,8 +1400,6 @@ workflows: - yellow-paper: *defaults - - boxes-files: *defaults - - noir-projects: requires: - avm-transpiler @@ -1427,28 +1407,28 @@ workflows: <<: *defaults # Yarn Project - - yarn-project-base: + - yarn-project-pre-join: requires: - bb-js - noir-packages - <<: *defaults - - yarn-project: - requires: - - yarn-project-base - l1-contracts - noir-projects - - boxes-files <<: *defaults - - yarn-project-prod: *defaults_yarn_project - - yarn-project-formatting: *defaults_yarn_project - - yarn-project-tests: *defaults_yarn_project - - end-to-end: *defaults_yarn_project - - build-docs: *defaults_yarn_project + - end-to-end: *defaults_yarn_project_pre_join + - aztec-faucet: *defaults_yarn_project_pre_join + - build-docs: *defaults_yarn_project_pre_join + - yarn-project-test: *defaults_yarn_project_pre_join + - yarn-project-x86_64: *defaults_yarn_project_pre_join + - yarn-project-arm64: *defaults_yarn_project_pre_join + - yarn-project-ecr-manifest: + requires: + - yarn-project-x86_64 + - yarn-project-arm64 + <<: *defaults # Artifacts - - aztec-package: *defaults_yarn_project_prod - - cli: *defaults_yarn_project_prod - - aztec-faucet: *defaults_yarn_project_prod + - aztec-package: *defaults_yarn_project + - cli: *defaults_yarn_project # Boxes. - boxes: diff --git a/.gitignore b/.gitignore index f24f1a84f27..e2fd5f4e513 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ cmake-build-debug .graphite* .DS_Store + +**/*.dockerignore \ No newline at end of file diff --git a/avm-transpiler/Dockerfile.dockerignore b/avm-transpiler/Dockerfile.dockerignore deleted file mode 100644 index 8edf9d12d62..00000000000 --- a/avm-transpiler/Dockerfile.dockerignore +++ /dev/null @@ -1,7 +0,0 @@ -** - -!avm-transpiler/ -!noir/ -**/target/ -**/node_modules/ -**/packages/ \ No newline at end of file diff --git a/barretenberg/.dockerignore b/barretenberg/.dockerignore deleted file mode 100644 index 2920dd00c6c..00000000000 --- a/barretenberg/.dockerignore +++ /dev/null @@ -1,11 +0,0 @@ -sol/broadcast -sol/cache -sol/out -sol/Dockerfile -sol/lib -sol/.foundry -sol/cache -sol/out -cpp/build -cpp/srs_db/ignition -.gitmodules \ No newline at end of file diff --git a/barretenberg/acir_tests/.dockerignore b/barretenberg/acir_tests/.dockerignore deleted file mode 100644 index 9dcf5756d6d..00000000000 --- a/barretenberg/acir_tests/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -acir_tests* -**/node_modules -Dockerfile* \ No newline at end of file diff --git a/barretenberg/cpp/.dockerignore b/barretenberg/cpp/.dockerignore deleted file mode 100644 index e9b3e4636d3..00000000000 --- a/barretenberg/cpp/.dockerignore +++ /dev/null @@ -1,27 +0,0 @@ -# We want to explicitly define what's allowed into the context. -* - -# Important cmake files. -!CMakeLists.txt -!CMakePresets.json -!cmake - -# Important srs_db files. -!srs_db/download_ignition.sh -!srs_db/download_grumpkin.sh -!srs_db/ignition/checksums - -# Source code. -!src/CMakeLists.txt -!src/barretenberg -!src/msgpack-c - -# Needed scripts. -!scripts/install-wasi-sdk.sh -!scripts/strip-wasm.sh -!scripts/ci -!./.clang-format -!./format.sh - -# Doxygen stuff. -!docs diff --git a/barretenberg/sol/Dockerfile b/barretenberg/sol/Dockerfile index bae96012f4c..df99298fb85 100644 --- a/barretenberg/sol/Dockerfile +++ b/barretenberg/sol/Dockerfile @@ -23,20 +23,14 @@ RUN apk update && apk add git curl build-base openmp-dev bash COPY --from=0 /usr/src/barretenberg/cpp/build/bin /usr/src/barretenberg/cpp/build/bin COPY --from=0 /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db WORKDIR /usr/src/barretenberg/sol -RUN git init COPY ./sol . # Copy forge binary directly from foundry COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/forge /usr/local/bin/forge -RUN forge install --no-commit \ - https://github.com/foundry-rs/forge-std \ - https://github.com/openzeppelin/openzeppelin-contracts \ - https://github.com/Arachnid/solidity-stringutils - RUN cd ../cpp/srs_db && ./download_ignition.sh 3 && cd ../../sol RUN ./scripts/init.sh -# TestBase is excluded as it is just boilerplate +# TestBase is excluded as it is just boilerplate RUN forge test --no-match-contract TestBase \ No newline at end of file diff --git a/barretenberg/ts/.dockerignore b/barretenberg/ts/.dockerignore deleted file mode 100644 index ab18ce85c3d..00000000000 --- a/barretenberg/ts/.dockerignore +++ /dev/null @@ -1,7 +0,0 @@ -dest -node_modules -.dockerignore -Dockerfile -.yarn -!.yarn/releases -.tsbuildinfo* \ No newline at end of file diff --git a/boxes/.dockerignore b/boxes/.dockerignore deleted file mode 100644 index 7eab5f5a5cb..00000000000 --- a/boxes/.dockerignore +++ /dev/null @@ -1,9 +0,0 @@ -.yarn/* -!.yarn/releases -dest -node_modules -.tsbuildinfo -Dockerfile* -.dockerignore -docker-compose.yml -**/artifacts \ No newline at end of file diff --git a/boxes/Dockerfile b/boxes/Dockerfile index dc1ca00f1bd..9c40953a0eb 100644 --- a/boxes/Dockerfile +++ b/boxes/Dockerfile @@ -1,4 +1,3 @@ -# Builds the boxes (they were copied into yarn-project-base so the cli can unbox). # Produces a container that can be run to test a specific box. See docker-compose.yml. FROM aztecprotocol/aztec AS aztec FROM aztecprotocol/noir as noir @@ -10,7 +9,9 @@ RUN apt update && apt install netcat-openbsd COPY --from=aztec /usr/src /usr/src COPY --from=noir /usr/src/noir/noir-repo/target/release/nargo /usr/src/noir/noir-repo/target/release/nargo COPY --from=noir-projects /usr/src/noir-projects/aztec-nr /usr/src/noir-projects/aztec-nr +COPY --from=noir-projects /usr/src/noir-projects/noir-protocol-circuits/crates/types /usr/src/noir-projects/noir-protocol-circuits/crates/types WORKDIR /usr/src/boxes +COPY . . ENV AZTEC_NARGO=/usr/src/noir/noir-repo/target/release/nargo ENV AZTEC_CLI=/usr/src/yarn-project/cli/aztec-cli-dest RUN yarn && yarn build diff --git a/boxes/Dockerfile.files b/boxes/Dockerfile.files deleted file mode 100644 index 411f2bf1bc7..00000000000 --- a/boxes/Dockerfile.files +++ /dev/null @@ -1,4 +0,0 @@ -# For carrying the files into yarn-project-base, needed for the cli to do unboxing. -FROM scratch -WORKDIR /usr/src/boxes -COPY . . \ No newline at end of file diff --git a/build-system/scripts/build b/build-system/scripts/build index 8b6ae7beb19..090878dd135 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -31,28 +31,6 @@ echo "Working directory: $PWD" echo "Dockerfile: $DOCKERFILE" echo "Build directory: $BUILD_DIR" -# Fetch images with retries -function fetch_image() { - echo "Pulling: $1" - if ! retry docker pull $1 > /dev/null 2>&1; then - echo "Image not found: $1" - return 1 - fi - return 0 -} - -# Fetch images, but don't assume this will work -function try_fetch_image() { - echo "Pulling: $1" - if ! docker pull $1 > /dev/null 2>&1; then - echo "Image not found: $1" - return 1 - fi - return 0 -} - -NOREPO=$(query_manifest noRepo $REPOSITORY) - # Login to ECR and ensure repository exists. retry ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle # Login to dockerhub. @@ -91,6 +69,7 @@ if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then fi # For each dependency, substitute references to the dependency in dockerfile, with the relevent built image uri. +# This is necessary vs pulling and retagging the image, as that doesn't work with buildx. # We have to perform a bit of probing to determine which actual image we want to use. # When we used buildx to create a multiarch image, there will be no images with "-$ARCH" suffixes (normalise this?). # Also we sometimes build an arm image from an x86 parent, so there won't always be an arm parent, and we fallback. @@ -115,9 +94,12 @@ for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do # Substitute references to parent repo, with the relevent built image uri. DEPLOY_URI=aztecprotocol/$PARENT_REPO PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:$PARENT_IMAGE_TAG - awk '{if ($1 == "FROM" && $2 == "'$DEPLOY_URI'") $2 = "'$PARENT_IMAGE_URI'"; print $0}' $DOCKERFILE > _temp && mv _temp $DOCKERFILE + sed -i "s#^FROM \\(.*\\)${DEPLOY_URI}\\( \|$\\)#FROM \\1${PARENT_IMAGE_URI}\\2#" $DOCKERFILE done +# Build a dockerignore file that only permits git files and untracked files. +create_dockerignore $REPOSITORY + COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false) echo "Commit tag version: $COMMIT_TAG_VERSION" diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index 41e24b0c23c..a737f3fb81e 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -40,6 +40,10 @@ for DEP in ${DEPS[@]}; do PROJECTS+=("$DEP:$BUILD_DIR:$DOCKERFILE") done +if [ -z "$NO_CACHE" ] && can_use_ci_cache; then + USE_CACHE=1 +fi + for E in "${PROJECTS[@]}"; do ARR=(${E//:/ }) PROJECT_DIR_NAME=${ARR[0]} @@ -65,6 +69,9 @@ for E in "${PROJECTS[@]}"; do fi fi + # Build a dockerignore file that only permits git files and untracked files. + create_dockerignore $REPO + DEPLOY_IMAGE_URI=aztecprotocol/$REPO:latest CACHE_IMAGE_URI=$(calculate_image_uri $REPO) ARR=(${CACHE_IMAGE_URI//:/ }) @@ -101,7 +108,7 @@ for E in "${PROJECTS[@]}"; do if [ -z "$NO_CACHE" ] && docker image ls --format "{{.Repository}}:{{.Tag}}" | grep -q -w "$CACHE_IMAGE_URI$"; then echo -e "${GREEN}Image exists locally. Tagging as $DEPLOY_IMAGE_URI${RESET}" else - if [ -z "$NO_CACHE" ] && [ -f ~/.aws/credentials ] && ecr_login && image_exists $REPO $TAG ; then + if [ -n "$USE_CACHE" ] && image_exists $REPO $TAG ; then docker pull $CACHE_IMAGE_URI else docker build ${ADDITIONAL_ARGS:-} --build-arg COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $CACHE_IMAGE_URI . diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 23a678fcd32..41fcffa93ad 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -35,6 +35,6 @@ echo "Success tag: $SUCCESS_TAG" if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then init_submodules $REPOSITORY - spot_run_script $SUCCESS_TAG $CPUS $ARCH $@ + spot_run_script $REPOSITORY $SUCCESS_TAG $CPUS $ARCH $@ retry tag_remote_image $REPOSITORY $BASE_TAG $SUCCESS_TAG fi diff --git a/build-system/scripts/create_dockerignore b/build-system/scripts/create_dockerignore new file mode 100755 index 00000000000..feade28a12f --- /dev/null +++ b/build-system/scripts/create_dockerignore @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Build a dockerignore file that only permits git files and untracked files. +# Exludes any docker related files. +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +REPOSITORY=$1 + +BUILD_DIR=$(query_manifest buildDir $REPOSITORY) +DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) + +cd $BUILD_DIR +DOCKERIGNOREFILE=$DOCKERFILE.dockerignore +echo '*' > $DOCKERIGNOREFILE +(git ls-files; git ls-files --others --exclude-standard) | sort -u | sed 's/^/!/' >> $DOCKERIGNOREFILE +echo '**/Dockerfile*' >> $DOCKERIGNOREFILE \ No newline at end of file diff --git a/build-system/scripts/ecr_login b/build-system/scripts/ecr_login index 5d9e6671e59..0d11088065b 100755 --- a/build-system/scripts/ecr_login +++ b/build-system/scripts/ecr_login @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail + REGION=${1:-$ECR_REGION} aws ecr get-login-password --region $REGION \ | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com 2> /dev/null \ No newline at end of file diff --git a/build-system/scripts/remove_old_images b/build-system/scripts/remove_old_images index 7c1c927e6b6..41e194ff8f6 100755 --- a/build-system/scripts/remove_old_images +++ b/build-system/scripts/remove_old_images @@ -11,3 +11,5 @@ for IMAGE in $(docker images --format "{{.ID}}" $ECR_URL/$REPOSITORY --filter "b echo "Removing $IMAGE..." docker rmi --force $IMAGE done + +docker image prune -f > /dev/null \ No newline at end of file diff --git a/build-system/scripts/spot_run_script b/build-system/scripts/spot_run_script index d5e77e35712..d36a2b249b7 100755 --- a/build-system/scripts/spot_run_script +++ b/build-system/scripts/spot_run_script @@ -8,10 +8,11 @@ # JOB_NAME: Set within setup-env. The job name as per CI. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu -NAME=$1 -CPUS=$2 -ARCH=$3 -shift 3 +REPOSITORY=$1 +NAME=$2 +CPUS=$3 +ARCH=$4 +shift 4 IP= # On any sort of exit (error or not). @@ -19,6 +20,11 @@ function on_exit { # We want to execute all this block, regardless of potential errors. set +e + if [[ "$COMMIT_MESSAGE" == *"[ci no-term $REPOSITORY]"* ]]; then + echo "Not terminating instance on request. You can connect with 'ssh -i build_instance_key ubuntu@$IP'" + return + fi + if [ -n "$IP" ]; then echo "Terminating spot instance..." ssh -F $SSH_CONFIG_PATH $IP sudo halt -p > /dev/null 2>&1 diff --git a/build_manifest.yml b/build_manifest.yml index 6100e90d66a..c2a3678dca4 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -142,24 +142,19 @@ barretenberg-docs: l1-contracts: buildDir: l1-contracts -# Just contains the source code for the boxes (starter kits). -# Needed to pull into yarn-project, as we wanted to keep them separate to yarn-project itself. -boxes-files: - buildDir: boxes - dockerfile: Dockerfile.files - -# Contains just the npm/js dependencies needed by yarn-project. -# It's its own thing, to avoid continuously downloading all dependencies every build. -yarn-project-base: +# Runs all yarn-projects checks and tests. +yarn-project-test: buildDir: yarn-project - projectDir: yarn-project/yarn-project-base + dockerfile: Dockerfile.test rebuildPatterns: - - ^yarn-project/yarn-project-base/ - - ^yarn-project/yarn.lock - - ^yarn-project/.*/package.json$ + - ^yarn-project/.*\.(ts|tsx|js|cjs|mjs|json|html|md|sh|nr|toml|snap)$ + - ^yarn-project/Dockerfile$ + - ^yarn-project/cli/aztec-cli dependencies: - bb.js - noir-packages + - l1-contracts + - noir-projects # Builds all of yarn-project, with all developer dependencies. # Creates a runnable container used to run tests and formatting checks. @@ -170,41 +165,34 @@ yarn-project: - ^yarn-project/Dockerfile$ - ^yarn-project/cli/aztec-cli dependencies: - - yarn-project-base + - bb.js + - noir-packages - l1-contracts - - boxes-files - noir-projects - -# Productionifies yarn-project (removes all dev dependencies, multiarch). -yarn-project-prod: - buildDir: yarn-project - dockerfile: Dockerfile.prod - rebuildPatterns: - - ^yarn-project/Dockerfile.prod - dependencies: - - yarn-project - multiarch: buildx + multiarch: host # A runnable container, sets entrypoint to be the aztec infrastructure entrypoint. aztec: buildDir: yarn-project projectDir: yarn-project/aztec dependencies: - - yarn-project-prod + - yarn-project multiarch: buildx +# Aztec faucet server. Has these dependencies because it's part of workspace. Consider moving out? aztec-faucet: buildDir: yarn-project projectDir: yarn-project/aztec-faucet dependencies: - - yarn-project-prod + - bb.js + - noir-packages # A runnable container, sets entrypoint to be the aztec-cli entrypoint. cli: buildDir: yarn-project projectDir: yarn-project/cli dependencies: - - yarn-project-prod + - yarn-project multiarch: buildx # Builds all the boxes. They are then independently tested in the container. @@ -217,12 +205,16 @@ boxes: runDependencies: - aztec -# Builds a runnable container for running end-to-end tests (requires installing puppeteer etc). +# Builds a runnable container for running end-to-end tests (requires installing chromium etc). +# We add the runnable dependency 'aztec' to ensure the container is pulled before running the test compose file. end-to-end: buildDir: yarn-project projectDir: yarn-project/end-to-end dependencies: - - yarn-project + - bb.js + - noir-packages + - l1-contracts + - noir-projects runDependencies: - aztec @@ -241,7 +233,9 @@ docs: - ^.release-please-manifest\.json$ - ^.*\.nr$ dependencies: - - yarn-project + - bb.js + - noir-packages + - l1-contracts - noir-projects yellow-paper: diff --git a/docs/.dockerignore b/docs/.dockerignore deleted file mode 100644 index b5f7883caaf..00000000000 --- a/docs/.dockerignore +++ /dev/null @@ -1,22 +0,0 @@ -**/node_modules -*.docusaurus -*.processed-docs -*.processed-docs-cache -*Dockerfile -*.CONTRIBUTING.md -*.LICENSE - -# Ignore C++ object files and executables -build/ -Debug/ -Release/ -barretenberg/ - -# Ignore Node.js build artifacts -*/node_modules/ -*.log -npm-debug.log -yarn.lock -yarn-error.log -package-lock.json -dist/ diff --git a/docs/Dockerfile b/docs/Dockerfile index d963bf86ec7..e9d9f58beec 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,5 +1,40 @@ -FROM aztecprotocol/yarn-project +FROM --platform=linux/amd64 aztecprotocol/bb.js as bb.js +FROM --platform=linux/amd64 aztecprotocol/noir-packages as noir-packages +FROM --platform=linux/amd64 aztecprotocol/l1-contracts as contracts +FROM --platform=linux/amd64 aztecprotocol/noir-projects as noir-projects + +FROM node:18.19.0 as builder +RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean + +# Copy in portalled packages. +COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts +COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages +COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts +COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects + WORKDIR /usr/src COPY . . + +# We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they +# walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution +# correctly for portalled packages, is to use --preserve-symlinks when running node. +# This does kind of work, but jest doesn't honor it correctly, so this seems like a neat workaround. +# Also, --preserve-symlinks causes duplication of portalled instances such as bb.js, and breaks the singleton logic +# by initialising the module more than once. So at present I don't see a viable alternative. +RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules + +# TODO: Replace puppeteer with puppeteer-core to avoid this. +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true + +WORKDIR /usr/src/yarn-project +RUN ./bootstrap.sh + WORKDIR /usr/src/docs -RUN yarn && yarn build \ No newline at end of file +RUN yarn && yarn build + +WORKDIR /usr/src/yarn-project +RUN yarn workspaces focus @aztec/scripts --production && yarn cache clean + +FROM node:18.19.1-slim +COPY --from=builder /usr/src/docs /usr/src/docs +COPY --from=builder /usr/src/yarn-project /usr/src/yarn-project \ No newline at end of file diff --git a/docs/Dockerfile.dockerignore b/docs/Dockerfile.dockerignore deleted file mode 100644 index cedb373cab3..00000000000 --- a/docs/Dockerfile.dockerignore +++ /dev/null @@ -1,14 +0,0 @@ -# The build context for docs is the root of the repository. -# Be very specific about what we include. -* - -!docs -docs/node_modules -!l1-contracts/src -!l1-contracts/test -!barretenberg/cpp/src/barretenberg -!.release-please-manifest.json -!boxes - -# Docs build fetches code snippets from the last release using git show. -!.git \ No newline at end of file diff --git a/docs/deploy_netlify.sh b/docs/deploy_netlify.sh index 1a1ff46a78c..aa38bf72b78 100755 --- a/docs/deploy_netlify.sh +++ b/docs/deploy_netlify.sh @@ -39,8 +39,6 @@ elif [ "$1" != "master" ]; then UNIQUE_DEPLOY_URL=$(echo "$DEPLOY_OUTPUT" | grep -E "https://.*aztec-docs-dev.netlify.app" | awk '{print $4}') echo "Unique deploy URL: $UNIQUE_DEPLOY_URL" - extract_repo yarn-project /usr/src project - cd project/src/yarn-project/scripts - + cd ../yarn-project/scripts UNIQUE_DEPLOY_URL=$UNIQUE_DEPLOY_URL yarn docs-preview-comment fi diff --git a/l1-contracts/.dockerignore b/l1-contracts/.dockerignore deleted file mode 100644 index 760149ef1b8..00000000000 --- a/l1-contracts/.dockerignore +++ /dev/null @@ -1,8 +0,0 @@ -.foundry -cache -Dockerfile -README.md -.github -lib/ -out/ -node_modules/ diff --git a/l1-contracts/Dockerfile b/l1-contracts/Dockerfile index bb0511f6657..671da76b806 100644 --- a/l1-contracts/Dockerfile +++ b/l1-contracts/Dockerfile @@ -4,11 +4,6 @@ RUN apk update && apk add git jq bash nodejs npm yarn python3 py3-pip && pip3 in WORKDIR /usr/src/l1-contracts COPY . . RUN git init -# Install deps -RUN forge install --no-commit \ - https://github.com/foundry-rs/forge-std \ - https://github.com/openzeppelin/openzeppelin-contracts -# Run build and tests RUN forge clean && forge fmt --check && forge build && forge test RUN yarn && yarn lint RUN git add . && yarn slither && yarn slither-has-diff diff --git a/noir-projects/.dockerignore b/noir-projects/.dockerignore deleted file mode 100644 index f01ca50f653..00000000000 --- a/noir-projects/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -aztec-nr/.gitattributes -aztec-nr/.gitrepo diff --git a/noir/.dockerignore b/noir/.dockerignore deleted file mode 100644 index f68bd5840b4..00000000000 --- a/noir/.dockerignore +++ /dev/null @@ -1,30 +0,0 @@ -**/Dockerfile* -**/.dockerignore - -# Yarn - -**/.pnp._ -**/.yarn/_ -**/!.yarn/patches -**/!.yarn/plugins -**/!.yarn/releases -**/!.yarn/sdks -**/!.yarn/versions - -packages -**/package.tgz -**/target -**/node_modules -**/outputs - -# Noir.js - -**/tooling/noir_js/lib - -# Wasm build artifacts - -**/compiler/wasm/nodejs -**/compiler/wasm/web -**/tooling/noirc_abi_wasm/nodejs -**/tooling/noirc_abi_wasm/web -**/tooling/noir_js/lib diff --git a/noir/noir-repo/.dockerignore b/noir/noir-repo/.dockerignore deleted file mode 100644 index 559b271bf38..00000000000 --- a/noir/noir-repo/.dockerignore +++ /dev/null @@ -1,27 +0,0 @@ -Dockerfile* -.dockerignore - -# Yarn -.pnp.* -.yarn/* -!.yarn/patches -!.yarn/plugins -!.yarn/releases -!.yarn/sdks -!.yarn/versions - -packages -**/package.tgz -**/target -**/node_modules -**/outputs - -# Noir.js -tooling/noir_js/lib - -# Wasm build artifacts -compiler/wasm/nodejs -compiler/wasm/web -tooling/noirc_abi_wasm/nodejs -tooling/noirc_abi_wasm/web -tooling/noir_js/lib diff --git a/yarn-project/.dockerignore b/yarn-project/.dockerignore deleted file mode 100644 index a3d11429d55..00000000000 --- a/yarn-project/.dockerignore +++ /dev/null @@ -1,17 +0,0 @@ -.yarn -!/.yarn/patches -!/.yarn/plugins -!/.yarn/releases -!/.yarn/sdks -!/.yarn/versions - -*/data* -**/dest -**/node_modules -**/Dockerfile* -**/*.tsbuildinfo - -./accounts/src/artifacts - -noir-contracts.js/src -noir-contracts.js/target diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index 9937be7aa9f..c7c30f83ada 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -1,13 +1,46 @@ -# This base dockerfile adds all the remaining source files and builds the project. -# See yarn-project-base/Dockerfile for why we have separate base Dockerfile. -FROM aztecprotocol/l1-contracts as contracts -FROM aztecprotocol/noir-projects as noir-projects -FROM aztecprotocol/boxes-files as boxes-files +FROM --platform=linux/amd64 aztecprotocol/bb.js as bb.js +FROM --platform=linux/amd64 aztecprotocol/noir-packages as noir-packages +FROM --platform=linux/amd64 aztecprotocol/l1-contracts as contracts +FROM --platform=linux/amd64 aztecprotocol/noir-projects as noir-projects -FROM aztecprotocol/yarn-project-base +FROM node:18.19.0 as builder +RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean + +# Copy in portalled packages. +COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts +COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects -COPY --from=boxes-files /usr/src/boxes /usr/src/boxes + +WORKDIR /usr/src/yarn-project COPY . . + +# We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they +# walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution +# correctly for portalled packages, is to use --preserve-symlinks when running node. +# This does kind of work, but jest doesn't honor it correctly, so this seems like a neat workaround. +# Also, --preserve-symlinks causes duplication of portalled instances such as bb.js, and breaks the singleton logic +# by initialising the module more than once. So at present I don't see a viable alternative. +RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules + +# TODO: Replace puppeteer with puppeteer-core to avoid this. +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true + RUN ./bootstrap.sh +RUN yarn workspaces focus @aztec/cli @aztec/aztec --production && yarn cache clean + +# TODO: Use release-please to update package.json directly, and remove this! +# It's here to ensure the image rebuilds if the commit tag changes (as the content hash won't). +ARG COMMIT_TAG="" +RUN ./scripts/version_packages.sh + +# We no longer need nargo etc. +RUN rm -rf /usr/src/noir/noir-repo /usr/src/noir-projects /usr/src/l1-contracts + +# Create minimal size image. +FROM node:18.19.1-slim +ARG COMMIT_TAG="" +ENV COMMIT_TAG=$COMMIT_TAG +COPY --from=builder /usr/src /usr/src +WORKDIR /usr/src/yarn-project ENTRYPOINT ["yarn"] diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod deleted file mode 100644 index bea13c08590..00000000000 --- a/yarn-project/Dockerfile.prod +++ /dev/null @@ -1,59 +0,0 @@ -# This productionifies the workspace, removing all developer dependencies and producing a final slim image from which -# we then generate downstream multiarch containers to execute the specific projects. -FROM aztecprotocol/yarn-project AS yarn-project - -# Need new arch specific image. -FROM node:18.19.0 AS builder -RUN apt update && apt install -y jq && rm -rf /var/lib/apt/lists/* && apt-get clean -COPY --from=yarn-project /usr/src /usr/src -WORKDIR /usr/src/yarn-project -# TODO: Use release-please to update package.json directly, and remove this! -ARG COMMIT_TAG="" -RUN ./scripts/version_packages.sh -# Productionify. See comment in yarn-project-base/Dockerfile. -RUN yarn workspaces focus @aztec/cli @aztec/aztec @aztec/aztec-faucet @aztec/aztec.js --production && yarn cache clean - -# We no longer need nargo. -RUN rm -rf /usr/src/noir/noir-repo/target - -# Create fresh minimal size image. -# Installs our specific version of node, stripping out the unnecessary. -# We could probably just apt install nodejs, but it's both a different version, and seemingly a bit slower. -# We could also use distroless, to get us about 20mb off, but meh. It's actually useful to shell into containers. -#FROM gcr.io/distroless/nodejs18-debian12 -FROM ubuntu:lunar -ARG COMMIT_TAG="" -ENV COMMIT_TAG=$COMMIT_TAG -# RUN apt update && apt install -y nodejs && rm -rf /var/lib/apt/lists/* && apt-get clean -RUN apt update && apt install -y curl && rm -rf /var/lib/apt/lists/* && apt-get clean -ENV NODE_VERSION=18.19.0 -RUN ARCH= && \ - dpkgArch="$(dpkg --print-architecture)" && \ - case "${dpkgArch##*-}" in \ - amd64) ARCH='x64';; \ - arm64) ARCH='arm64';; \ - *) echo "unsupported architecture"; exit 1 ;; \ - esac && \ - curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.gz" && \ - tar zxf "node-v$NODE_VERSION-linux-$ARCH.tar.gz" -C /usr --strip-components=1 --no-same-owner \ - --exclude "*/share/*" \ - --exclude "*/bin/corepack" \ - --exclude "*/bin/npx" \ - --exclude "*/bin/npm" \ - --exclude "*/corepack/*" \ - --exclude "*/npm/man/*" \ - --exclude "*/npm/docs/*" \ - --exclude "*/include/*" && \ - rm "node-v$NODE_VERSION-linux-$ARCH.tar.gz" && \ - node --version -# Yarn is used for unboxing. -ENV YARN_VERSION=1.22.19 -RUN curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" && \ - mkdir -p /opt && \ - tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ && \ - ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn && \ - ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg && \ - rm yarn-v$YARN_VERSION.tar.gz && \ - yarn --version -COPY --from=builder /usr/src /usr/src -ENTRYPOINT ["/usr/bin/node"] diff --git a/yarn-project/Dockerfile.test b/yarn-project/Dockerfile.test new file mode 100644 index 00000000000..6dfe4510760 --- /dev/null +++ b/yarn-project/Dockerfile.test @@ -0,0 +1,34 @@ +FROM --platform=linux/amd64 aztecprotocol/bb.js as bb.js +FROM --platform=linux/amd64 aztecprotocol/noir-packages as noir-packages +FROM --platform=linux/amd64 aztecprotocol/l1-contracts as contracts +FROM --platform=linux/amd64 aztecprotocol/noir-projects as noir-projects + +FROM node:18.19.0 as builder +RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean + +# Copy in portalled packages. +COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts +COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages +COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts +COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects + +WORKDIR /usr/src/yarn-project +COPY . . + +# We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they +# walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution +# correctly for portalled packages, is to use --preserve-symlinks when running node. +# This does kind of work, but jest doesn't honor it correctly, so this seems like a neat workaround. +# Also, --preserve-symlinks causes duplication of portalled instances such as bb.js, and breaks the singleton logic +# by initialising the module more than once. So at present I don't see a viable alternative. +RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules + +# TODO: Replace puppeteer with puppeteer-core to avoid this. +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true + +RUN ./bootstrap.sh +RUN yarn prepare:check && yarn formatting && yarn test + +# Avoid pushing some huge container back to ecr. +FROM scratch +COPY --from=builder /usr/src/yarn-project/README.md /usr/src/yarn-project/README.md \ No newline at end of file diff --git a/yarn-project/aztec-faucet/.dockerignore b/yarn-project/aztec-faucet/.dockerignore deleted file mode 100644 index 2b30eaf4896..00000000000 --- a/yarn-project/aztec-faucet/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -data -dest -node_modules -Dockerfile \ No newline at end of file diff --git a/yarn-project/aztec-faucet/Dockerfile b/yarn-project/aztec-faucet/Dockerfile index 047f7cd0cac..f0043d91fca 100644 --- a/yarn-project/aztec-faucet/Dockerfile +++ b/yarn-project/aztec-faucet/Dockerfile @@ -1,4 +1,25 @@ -FROM aztecprotocol/yarn-project-prod +FROM --platform=linux/amd64 aztecprotocol/bb.js as bb.js +FROM --platform=linux/amd64 aztecprotocol/noir-packages as noir-packages + +FROM node:18.19.0 as builder + +# Copy in portalled packages. +# We need them because faucet is part of the workspace, and the workspace depends on them. 🤷‍♂️ +COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts +COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages + +WORKDIR /usr/src/yarn-project +COPY . . +RUN yarn workspaces focus @aztec/aztec-faucet WORKDIR /usr/src/yarn-project/aztec-faucet +RUN yarn tsc -b +RUN yarn workspaces focus @aztec/aztec-faucet --production && yarn cache clean + +# Create minimal image. +FROM ubuntu:lunar +ARG COMMIT_TAG="" +ENV COMMIT_TAG=$COMMIT_TAG +RUN apt update && apt install -y nodejs && rm -rf /var/lib/apt/lists/* && apt-get clean +COPY --from=builder /usr/src/yarn-project /usr/src/yarn-project ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/aztec-faucet/dest/bin/index.js"] EXPOSE 8080 diff --git a/yarn-project/aztec/Dockerfile b/yarn-project/aztec/Dockerfile index 13b2000744d..ae996a59dfb 100644 --- a/yarn-project/aztec/Dockerfile +++ b/yarn-project/aztec/Dockerfile @@ -1,7 +1,7 @@ -FROM aztecprotocol/yarn-project-prod AS yarn-project-prod +FROM aztecprotocol/yarn-project AS yarn-project ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/aztec/dest/bin/index.js"] EXPOSE 8080 -# The version has been updated in yarn-project-prod. +# The version has been updated in yarn-project. # Adding COMMIT_TAG here to rebuild versioned image. ARG COMMIT_TAG="" \ No newline at end of file diff --git a/yarn-project/cli/Dockerfile b/yarn-project/cli/Dockerfile index b782d46e801..c69606b278c 100644 --- a/yarn-project/cli/Dockerfile +++ b/yarn-project/cli/Dockerfile @@ -1,7 +1,7 @@ -FROM aztecprotocol/yarn-project-prod AS yarn-project-prod +FROM aztecprotocol/yarn-project AS yarn-project ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/cli/dest/bin/index.js"] -# The version has been updated in yarn-project-prod. +# The version has been updated in yarn-project. # Adding COMMIT_TAG here to rebuild versioned image. ARG COMMIT_TAG="" diff --git a/yarn-project/deploy_npm.sh b/yarn-project/deploy_npm.sh index 04bf9c6e4d3..65ea41a7edd 100755 --- a/yarn-project/deploy_npm.sh +++ b/yarn-project/deploy_npm.sh @@ -8,7 +8,7 @@ if [ -z "$COMMIT_TAG" ]; then fi retry ecr_login -extract_repo yarn-project-prod /usr/src project +extract_repo yarn-project /usr/src project cd project/src/yarn-project echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >.npmrc diff --git a/yarn-project/end-to-end/Dockerfile b/yarn-project/end-to-end/Dockerfile index 49f5a14711d..e1934cff7b2 100644 --- a/yarn-project/end-to-end/Dockerfile +++ b/yarn-project/end-to-end/Dockerfile @@ -1,29 +1,49 @@ -FROM aztecprotocol/yarn-project AS builder +FROM --platform=linux/amd64 aztecprotocol/bb.js as bb.js +FROM --platform=linux/amd64 aztecprotocol/noir-packages as noir-packages +FROM --platform=linux/amd64 aztecprotocol/l1-contracts as contracts +FROM --platform=linux/amd64 aztecprotocol/noir-projects as noir-projects + +FROM node:18.19.0 as builder +RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean + +# Copy in portalled packages. +COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts +COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages +COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts +COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects + +WORKDIR /usr/src/yarn-project +COPY . . + +# We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they +# walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution +# correctly for portalled packages, is to use --preserve-symlinks when running node. +# This does kind of work, but jest doesn't honor it correctly, so this seems like a neat workaround. +# Also, --preserve-symlinks causes duplication of portalled instances such as bb.js, and breaks the singleton logic +# by initialising the module more than once. So at present I don't see a viable alternative. +RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules + +# TODO: Replace puppeteer with puppeteer-core to avoid this. +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true + +RUN ./bootstrap.sh # Build web bundle for browser tests -WORKDIR /usr/src/yarn-project/end-to-end -RUN yarn build:web - -# Productionify. See comment in yarn-project-base/Dockerfile. -RUN yarn workspaces focus --production && yarn cache clean - -# Create final, minimal size image. -# TODO: Not very minimal as chromium adds about 500MB of bloat :/ Separate or install at test runtime? -FROM node:18.19.0-alpine -RUN apk update && apk add --no-cache \ - jq \ - bash \ - chromium \ - chromium-chromedriver \ - nss \ - freetype \ - harfbuzz \ - ca-certificates \ - ttf-freefont - -ENV CHROME_BIN="/usr/bin/chromium-browser" PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true" -COPY --from=builder /usr/src /usr/src +RUN yarn workspace @aztec/end-to-end run build:web +RUN yarn workspaces focus @aztec/end-to-end --production && yarn cache clean -WORKDIR /usr/src/yarn-project/end-to-end +# We no longer need nargo etc. +RUN rm -rf /usr/src/noir/noir-repo /usr/src/noir-projects /usr/src/l1-contracts -ENTRYPOINT ["yarn", "test"] +# Create minimal image. +FROM node:18.19.1-slim +RUN apt-get update && apt-get install jq gnupg wget -y && \ + wget --quiet --output-document=- https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor > /etc/apt/trusted.gpg.d/google-archive.gpg && \ + sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' && \ + apt-get update && \ + apt-get install google-chrome-stable -y --no-install-recommends && \ + rm -rf /var/lib/apt/lists/* +ENV CHROME_BIN="/usr/bin/google-chrome-stable" +COPY --from=builder /usr/src /usr/src +WORKDIR /usr/src/yarn-project/end-to-end +ENTRYPOINT ["yarn", "test"] \ No newline at end of file diff --git a/yarn-project/end-to-end/scripts/docker-compose-browser.yml b/yarn-project/end-to-end/scripts/docker-compose-browser.yml deleted file mode 100644 index f859de7de39..00000000000 --- a/yarn-project/end-to-end/scripts/docker-compose-browser.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: '3' -services: - fork: - image: ghcr.io/foundry-rs/foundry:nightly-a44aa13cfc23491ba32aaedc093e9488c1a6db43 - entrypoint: > - sh -c ' - if [ -n "$FORK_BLOCK_NUMBER" ] && [ -n "$FORK_URL" ]; then - exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" - else - exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 - fi' - ports: - - '8545:8545' - - sandbox: - image: aztecprotocol/aztec:latest - environment: - DEBUG: 'aztec:*' - DEBUG_COLORS: 1 - ETHEREUM_HOST: http://fork:8545 - CHAIN_ID: 31337 - ARCHIVER_POLLING_INTERVAL_MS: 50 - P2P_BLOCK_CHECK_INTERVAL_MS: 50 - SEQ_TX_POLLING_INTERVAL_MS: 50 - WS_BLOCK_CHECK_INTERVAL_MS: 50 - PXE_BLOCK_POLLING_INTERVAL_MS: 50 - ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 - ports: - - '8080:8080' - - end-to-end: - image: aztecprotocol/end-to-end:latest - environment: - BENCHMARK: 'true' - DEBUG: ${DEBUG:-'aztec:*'} - DEBUG_COLORS: 1 - ETHEREUM_HOST: http://fork:8545 - CHAIN_ID: 31337 - PXE_URL: http://sandbox:8080 - entrypoint: ['./scripts/start_e2e_ci_browser.sh', './src/e2e_aztec_js_browser.test.ts'] - volumes: - - ../log:/usr/src/yarn-project/end-to-end/log:rw - depends_on: - - sandbox - - fork diff --git a/yarn-project/end-to-end/scripts/start_e2e_ci_browser.sh b/yarn-project/end-to-end/scripts/start_e2e_ci_browser.sh deleted file mode 100755 index f73db85b3fe..00000000000 --- a/yarn-project/end-to-end/scripts/start_e2e_ci_browser.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -TEST=${1:-./src/e2e_aztec_js_browser.test.ts} - -apk add dbus - -# Create dbus dirs -mkdir -p /var/run/dbus - -# Change ownership and permissions if necessary -chown -R root:root /var/run/dbus -chmod -R 755 /var/run/dbus - -dbus-daemon --system --nofork & -yarn test $TEST diff --git a/yarn-project/package.json b/yarn-project/package.json index ba737a9dd77..c2cb5c60bc3 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -3,14 +3,14 @@ "packageManager": "yarn@3.6.3", "private": true, "scripts": { - "prepare": "node ./yarn-project-base/scripts/update_package_jsons.mjs && yarn workspaces foreach run prepare && workspaces-to-typescript-project-references --tsconfigPath tsconfig.json && prettier -w */tsconfig.json", - "prepare:check": "node ./yarn-project-base/scripts/update_package_jsons.mjs --check && workspaces-to-typescript-project-references --check --tsconfigPath tsconfig.json", + "prepare": "node ./scripts/update_package_jsons.mjs && yarn workspaces foreach run prepare && workspaces-to-typescript-project-references --tsconfigPath tsconfig.json && prettier -w */tsconfig.json", + "prepare:check": "node ./scripts/update_package_jsons.mjs --check && workspaces-to-typescript-project-references --check --tsconfigPath tsconfig.json", "docs": "typedoc --out docs/dist && cd docs && yarn serve", - "formatting": "FORCE_COLOR=true yarn workspaces foreach -p -j unlimited -v run formatting", + "formatting": "FORCE_COLOR=true yarn workspaces foreach -p -v run formatting", "formatting:fix": "FORCE_COLOR=true yarn workspaces foreach -p -v run formatting:fix", "lint": "yarn eslint --cache --ignore-pattern l1-artifacts .", "format": "yarn prettier --cache -w .", - "test": "FORCE_COLOR=true yarn workspaces foreach --exclude @aztec/aztec3-packages --exclude @aztec/end-to-end -p -j ${JOBS:-unlimited} -v run test", + "test": "FORCE_COLOR=true yarn workspaces foreach --exclude @aztec/aztec3-packages --exclude @aztec/end-to-end -p -v run test", "build": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose --exclude @aztec/aztec3-packages --exclude @aztec/docs run build", "build:fast": "yarn generate && tsc -b", "build:dev": "tsc -b tsconfig.json --watch", diff --git a/yarn-project/yarn-project-base/scripts/update_package_jsons.mjs b/yarn-project/scripts/update_package_jsons.mjs similarity index 100% rename from yarn-project/yarn-project-base/scripts/update_package_jsons.mjs rename to yarn-project/scripts/update_package_jsons.mjs diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile deleted file mode 100644 index 62c4b55fb42..00000000000 --- a/yarn-project/yarn-project-base/Dockerfile +++ /dev/null @@ -1,82 +0,0 @@ -# This base Dockerfile is for: -# - Caching the workspace dependencies. -# - Running workspace checks (package.json inheritance, tsconfig.json project references). -# - Bringing in any upstream resources such as L1 contract artifacts and bb.js. -# - Performing any code generation that doesn't require the workspace code to do so (generate L1 artifacts). -# -# When installing workspace dependencies, there are issues with doing this naively: -# - We only ever want to re-download and install workspace dependencies in the event that they change. -# If a developer only changes some code, we want this step of the build to become a noop and to reuse existing image. -# Dockerfile.dockerignore is tailored to specifically only bring in files needed to install the dependencies. -# NOTE: For this dockerignore file to be used, you MUST be using BUILDKIT in more recent versions of Docker. -# Best to make sure you're on docker >= 24. On mainframe run `restart_docker`, it should auto-upgrade. -# - We want to disable yarn from accessing the net after initial package download as it prevents a class of build bugs. -# - We want to prune dev dependencies as this can significantly reduce the final container size even further. -# Yarn devs won't provide the ability to prune dev dependencies from the local project cache: -# https://github.com/yarnpkg/berry/issues/1789 -# This means we need a global cache, so we can clean the cache and reinstall prod modules without re-downloading. -# - The default global cache and local cache are simply copies, not hardlinks, thus doubling the storage of modules. -# To work around, we will construct a global cache from the local cache using hard links (requires a hacky rename). -# We do this in the same layer the original file is created, otherwise overlayfs creates a copy anyway. -# At time of writing this shaves off around 150MB. Not a big deal but a harmless trick for now. -# -# So, here in the base image we yarn install. -# - /root/.yarn/berry/cache (global cache) is populated with zipped packages. -# - /usr/src/yarn-project/.yarn/cache (project local cache) is populated with copy of zipped packages. -# - Packages are unzipped into various node_modules folders. -# - We then erase the global cache, and recreate each file as a hard link, reducing the zipped package storage by half. -# -# That's all we want to do here r.e. dependency installation. In yarn-project we will: -# - Copy in the rest of the workspace files. -# - Perform any code generation steps that require the workspace code (generate L2 contract wrappers). -# - Build all the TypeScript. -# -# When we build a downstream docker image, we: -# - FROM yarn-project as builder, to get the entire built project. -# - WORKDIR into the relevant project. -# - Do any project specific build work. -# - Do a `yarn workspaces focus --production` to install production dependencies from the global. -# - Erase the local cache with a `yarn cache clean`. -# - Destroy the yarn-project /src folders to save additional space. -# - The above can be done with: -# RUN yarn workspaces focus --production && yarn cache clean && rm -rf ../**/src -# - Create final slim image by copying needed dirs into a fresh image. -# -FROM aztecprotocol/bb.js as bb.js -FROM aztecprotocol/noir-packages as noir-packages - -FROM node:18.19.0 -RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean - -# Copy in portalled packages. -COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts -COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages - -# We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they -# walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution -# correctly for portalled packages, is to use --preserve-symlinks when running node. -# This does kind of work, but jest doesn't honor it correctly, so this seems like a neat workaround. -# Also, --preserve-symlinks causes duplication of portalled instances such as bb.js, and breaks the singleton logic -# by initialising the module more than once. So at present I don't see a viable alternative. -RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules - -WORKDIR /usr/src/yarn-project -# The dockerignore file ensures the context only contains package.json and tsconfig.json files. -COPY . . - -# List all included files and hash for debugging. -RUN echo "Context files: " && find . -type f | sort && \ - echo -n "Context hash: " && find . -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum - -# Install packages and rebuild the global cache with hard links. -# TODO: Puppeteer is adding ~300MB to this image due to chrome download (part of e2e). -# Switch to using puppeteer-core then it won't download chrome. For now just erase. -RUN yarn --immutable && rm -rf /root/.cache/puppeteer && /bin/bash -c '\ - rm -rf /root/.yarn/berry/cache/* && \ - cd .yarn/cache && \ - for F in *; do \ - [[ $F =~ (.*-) ]] && ln $F /root/.yarn/berry/cache/${BASH_REMATCH[1]}8.zip; \ - done' - -# Check package.json inheritance and tsconfig project references. -RUN yarn prepare:check \ No newline at end of file diff --git a/yarn-project/yarn-project-base/Dockerfile.dockerignore b/yarn-project/yarn-project-base/Dockerfile.dockerignore deleted file mode 100644 index b388fe69533..00000000000 --- a/yarn-project/yarn-project-base/Dockerfile.dockerignore +++ /dev/null @@ -1,29 +0,0 @@ -# The aim here is to not have to maintain a list of projects in either the Dockerfile or this ignore file. -# We need to be careful not to exclude the project directories, as you cannot then use a wildcard in a negation -# to re-include needed files. Here we use */* which doesn't exclude the directories themselves, but rather -# the files within those directories. As we haven't excluded anything in the root, we manually exclude root files. -*/* -.* -README.md -bootstrap.sh -Dockerfile* -*.tsbuildinfo -node_modules - -# Unexclude package.json and yarn.lock files, for detecting any dependency changes. -!*/package.json -!*/package.*.json -!yarn.lock - -# Unexclude parts of yarn related config as this also affects how dependencies are installed. -!.yarnrc.yml -!.yarn/plugins -!.yarn/releases -!.yarn/patches - -# Unexclude tsconfig files for running project reference checks. -!*/tsconfig.json - -# Unexclude scripts we use in the Dockerfile. -!yarn-project-base/scripts -!l1-artifacts/scripts \ No newline at end of file diff --git a/yarn-project/yarn-project-base/Dockerfile.dockerignore.v24 b/yarn-project/yarn-project-base/Dockerfile.dockerignore.v24 deleted file mode 100644 index 0627db619ac..00000000000 --- a/yarn-project/yarn-project-base/Dockerfile.dockerignore.v24 +++ /dev/null @@ -1,37 +0,0 @@ -# TODO/WARNING: ADOPT THIS WHEN WE HAVE DOCKER >= 24 IN NEXT UBUNTU LTS RELEASE. -# -# The aim here is to not have to maintain a list of projects in either the Dockerfile or this ignore file. -# This context should only contain precisely what's needed. -# -# This took a fair bit of trial and error to get the right result. If you need to meddle, try: -# - Commenting out everything after the COPY . . in the dockerfile. -# - ONLY_TARGET=1 ../bootstrap_docker.sh yarn-project-base && docker run -ti --rm aztecprotocol/yarn-project-base:latest sh -c 'du -ha .' - -# Exclude everything to start. -* - -# Unexclude package.json, tsconfig.json and yarn.lock files. -!package.json -!package.common.json -!*/package.json -!*/package.local.json -!*/*/package.json -!yarn.lock -!tsconfig.json -!*/tsconfig.json -!*/*/tsconfig.json - -# Unexclude parts of yarn related config as this also affects how dependencies are installed. -!.yarnrc.yml -!.yarn/plugins -!.yarn/releases -!.yarn/patches - -# Unexclude scripts we use in the Dockerfile. -!yarn-project-base/scripts -!l1-artifacts/scripts - -# Re-exclude any node_modules stuff (matters when building locally when you have a node_modules). -# Yes, we need to explicitly exclude what we unexcluded above. Just putting node_modules doesn't help here. -node_modules/*/package.json -node_modules/*/tsconfig.json diff --git a/yarn-project/yarn-project-base/README.md b/yarn-project/yarn-project-base/README.md deleted file mode 100644 index 2ed7ea6ef87..00000000000 --- a/yarn-project/yarn-project-base/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# yarn-project-base - -## Why? - -If you want to rebuild a docker container for a project in the workspace, you don't want to have to be waiting -to download the entire set of workspace dependencies just because you changed a line of code. The way docker caches -and reuses layers is very powerful. We build this base image in order to: - -1. Encapsulate the downloading of all workspace dependencies. -1. Check our package.json files have inherited from the common package.json. -1. Check out tsconfig project references are all correct. -1. Generate L1 contract ABIs. - -The root project Dockerfile `yarn-project` then: - -1. Generates Noir contract ABIs. -1. Builds the entire project. -1. Checks all formatting is correct. -1. Runs all workspace unit tests. - -Downstream projects are then just about containerizing what's needed to produce executable containers for e2e testing or -deployments. - -## Do we care about docker layer caching, when build-system rebuild patterns only trigger on yarn.lock changes? - -Enough. When building the containers locally for development or debugging purposes, you can't use the content hash -to skip parts of the build, as content hashes require everything to have been committed to git. This is usually -is not the case during development. diff --git a/yarn-project/yarn-project-base/package.json b/yarn-project/yarn-project-base/package.json deleted file mode 100644 index d0615316a9d..00000000000 --- a/yarn-project/yarn-project-base/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "@aztec/yarn-project-base", - "version": "0.0.0", - "type": "module" -} diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 14e47df91ba..ba211032989 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -946,12 +946,6 @@ __metadata: languageName: unknown linkType: soft -"@aztec/yarn-project-base@workspace:yarn-project-base": - version: 0.0.0-use.local - resolution: "@aztec/yarn-project-base@workspace:yarn-project-base" - languageName: unknown - linkType: soft - "@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.22.13, @babel/code-frame@npm:^7.23.5": version: 7.23.5 resolution: "@babel/code-frame@npm:7.23.5"