From 51352ae3973c937bbb6a4baee401aff52b54246d Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Tue, 26 Sep 2023 22:15:16 +0100 Subject: [PATCH] chore: Disable pushing/pulling for layer caching in build. (#2517) * There is little gain to pushing and pulling previously built images to leverage the layer cache, so don't. * If we want to avoid reinstalling compilers etc we should create better base images (later) * Docs doesn't need to build yarn-project anymore as base image already builds it. * Use query_manifest dependencies to determine dependents to pull and tag, rather than sedding Dockerfile. --------- Co-authored-by: ludamad --- build-system/scripts/build | 63 ++++------------------------- build-system/scripts/build_local | 2 +- build-system/scripts/query_manifest | 5 ++- build-system/scripts/setup_env | 2 +- build_manifest.yml | 22 +++++----- docs/Dockerfile | 4 +- docs/Dockerfile.dockerignore | 10 +++++ 7 files changed, 36 insertions(+), 72 deletions(-) create mode 100644 docs/Dockerfile.dockerignore diff --git a/build-system/scripts/build b/build-system/scripts/build index 5c077a2e4f4..e1a23cc8614 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -58,9 +58,7 @@ function try_fetch_image() { return 0 } -ecr_login - -# Ensure ECR repository exists. +# Login to ECR and ensure repository exists. retry ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle CONTENT_HASH=$(calculate_content_hash $REPOSITORY) @@ -78,9 +76,9 @@ fi echo "Initializing submodules..." init_submodules $REPOSITORY -echo "Checking for terraform..." # Validate any terraform if it exists. if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then + echo "Checking terraform..." ensure_terraform export TF_IN_AUTOMATION=1 pushd $ROOT_PATH/$PROJECT_DIR/terraform @@ -95,18 +93,8 @@ if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then popd fi -# Pull latest parents that are not ours. We also do not want to pull images suffixed by _, this is how we scope intermediate build images. -PARENTS=$(cat $DOCKERFILE | sed -n -e 's/^FROM \([^[:space:]]\+\).*/\1/p' | sed '/_$/d' | { grep -v $ECR_DEPLOY_URL || true; } | sort | uniq) -for PARENT in $PARENTS; do - [ "$PARENT" == "scratch" ] && continue - fetch_image $PARENT -done - -# For each parent that's ours, pull in the latest image. -PARENTS=$(cat $DOCKERFILE | sed -n -e "s/^FROM $ECR_DEPLOY_URL\/\([^[:space:]]\+\).*/\1/p") -for PARENT in $PARENTS; do - # Extract repository name (i.e. discard tag). - PARENT_REPO=${PARENT%:*} +# For each dependency, pull in the latest image and give it correct tag. +for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do PARENT_CONTENT_HASH=$(calculate_content_hash $PARENT_REPO) # There must be a parent image to continue. if [ -z "$PARENT_CONTENT_HASH" ]; then @@ -117,49 +105,12 @@ for PARENT in $PARENTS; do echo "Pulling dependency $PARENT_REPO..." fetch_image $PARENT_IMAGE_URI # Tag it to look like an official release as that's what we use in Dockerfiles. - TAG=$ECR_DEPLOY_URL/$PARENT + TAG=$ECR_DEPLOY_URL/$PARENT_REPO retry docker tag $PARENT_IMAGE_URI $TAG done -echo "Extracting commit tag version..." COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false) - -# Pull, build and push each named stage to cache. -STAGE_CACHE_FROM="" -CACHE_FROM="" -STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') -for STAGE in $STAGES; do - # Get the last build of this stage to leverage layer caching. - if [ -n "$CONTENT_HASH" ]; then - echo "Pulling stage: $STAGE" - STAGE_IMAGE_LAST_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$STAGE - if try_fetch_image $STAGE_IMAGE_LAST_URI; then - STAGE_CACHE_FROM="--cache-from $STAGE_IMAGE_LAST_URI" - fi - fi - - echo "Building stage: $STAGE" - STAGE_IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$STAGE - # Build our dockerfile, add timing information - docker build --target $STAGE $STAGE_CACHE_FROM -t $STAGE_IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . - - # We don't want to have redo this stages work when building the final image. Use it as a layer cache. - CACHE_FROM="--cache-from $STAGE_IMAGE_COMMIT_URI $CACHE_FROM" - - echo "Pushing stage: $STAGE" - retry docker push $STAGE_IMAGE_COMMIT_URI > /dev/null 2>&1 - echo -done - -# Pull previous image to use it as a layer cache if it exists, and if we have not manually ran. -if [ -n "$COMMIT_HASH" ]; then - LAST_SUCCESSFUL_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH - echo "Pulling previous build of $REPOSITORY..." - if try_fetch_image $LAST_SUCCESSFUL_URI ; then - CACHE_FROM="--cache-from $LAST_SUCCESSFUL_URI $CACHE_FROM" - fi - echo -fi +echo "Commit tag version: $COMMIT_TAG_VERSION" # Build the actual image and give it a commit tag. IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH @@ -167,6 +118,6 @@ if [[ -n "$ARCH" ]]; then IMAGE_COMMIT_URI=$IMAGE_COMMIT_URI-$ARCH fi echo "Building image: $IMAGE_COMMIT_URI" -docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE $CACHE_FROM --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . +docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . echo "Pushing image: $IMAGE_COMMIT_URI" retry docker push $IMAGE_COMMIT_URI > /dev/null 2>&1 diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index 75f0d07394b..f95c2413fe9 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -29,7 +29,7 @@ fi if [ -n "$ONLY_TARGET" ]; then DEPS=("$TARGET_PROJECT") else - DEPS=($(query_manifest dependencies $TARGET_PROJECT)) + DEPS=($(query_manifest allDependencies $TARGET_PROJECT)) fi declare -a PROJECTS for DEP in ${DEPS[@]}; do diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index 6c527012f5d..edf5f62b6fd 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -73,6 +73,9 @@ case "$CMD" in yq -r ".\"$REPO\".projectDir // .\"$REPO\".buildDir" $MANIFEST ;; dependencies) + yq -r ".\"$REPO\".dependencies // [] | .[]" $MANIFEST + ;; + allDependencies) # Get dependencies for a given repo. Inclusive of repo itself. # If a string, attempt to execute / if exists to get dependencies, else error. # If an array, the array lists the dependencies (excluding self). @@ -102,7 +105,7 @@ case "$CMD" in # If no rebuild patterns are given, the result is ["^/"]. # If a projects rebuildPattern is a string, the rebuild patterns are in /. # If an array, the array lists the rebuild patterns. - DEPS=($($0 dependencies $REPO)) + DEPS=($($0 allDependencies $REPO)) PATTERNS=() for DEP in "${DEPS[@]}"; do add_rebuild_patterns $DEP diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 29319f1b1b5..35553c37dcc 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -81,7 +81,7 @@ echo export ECR_DEPLOY_URL=278380418400.dkr.ecr.eu-west-2.amazonaws.com >> $BASH echo export PROJECT=$PROJECT >> $BASH_ENV echo export COMMIT_HASH=$COMMIT_HASH >> $BASH_ENV echo export COMMIT_TAG=$COMMIT_TAG >> $BASH_ENV -echo "export COMMIT_MESSAGE='$COMMIT_MESSAGE'" >> $BASH_ENV +echo "export COMMIT_MESSAGE='${COMMIT_MESSAGE//\'/\'\\\'\'}'" >> $BASH_ENV echo export JOB_NAME=$JOB_NAME >> $BASH_ENV echo export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL >> $BASH_ENV echo export VERSION_TAG=$VERSION_TAG >> $BASH_ENV diff --git a/build_manifest.yml b/build_manifest.yml index f972ade3336..b924b3d3941 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -75,16 +75,6 @@ circuits-x86_64-linux-clang-assert: dependencies: - barretenberg-x86_64-linux-clang -docs: - buildDir: . - dockerfile: docs/Dockerfile - rebuildPatterns: - - ^docs/ - - ^.*.cpp$ - - ^.*.ts$ - - ^.release-please-manifest.json$ - - ^.*/noir-version.json$ - l1-contracts: buildDir: l1-contracts @@ -167,3 +157,15 @@ end-to-end: projectDir: yarn-project/end-to-end dependencies: - yarn-project + +docs: + buildDir: . + dockerfile: docs/Dockerfile + rebuildPatterns: + - ^docs/ + - ^.*.cpp$ + - ^.*.ts$ + - ^.release-please-manifest.json$ + - ^.*/noir-version.json$ + dependencies: + - yarn-project diff --git a/docs/Dockerfile b/docs/Dockerfile index 9653ac4b2a6..847379f1ccf 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,7 +1,5 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS builder +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project WORKDIR /usr/src COPY . . -WORKDIR /usr/src/yarn-project -RUN yarn build WORKDIR /usr/src/docs RUN yarn && yarn build \ No newline at end of file diff --git a/docs/Dockerfile.dockerignore b/docs/Dockerfile.dockerignore new file mode 100644 index 00000000000..224a414b0af --- /dev/null +++ b/docs/Dockerfile.dockerignore @@ -0,0 +1,10 @@ +# The build context for docs is the root of the repository. +# Be very specific about what we include. +* + +!docs +!l1-contracts/src +!l1-contracts/test +!barretenberg/cpp/src/barretenberg +!circuits/cpp/src +!.release-please-manifest.json \ No newline at end of file