From 7d48ed3beb70f0ed183407e87dad0fb9310fcf13 Mon Sep 17 00:00:00 2001
From: spypsy <spypsy@users.noreply.github.com>
Date: Fri, 13 Oct 2023 10:59:52 +0100
Subject: [PATCH] feat: NPM canary deployment (#2731)

Fixes #2366

# Checklist:
Remove the checklist to signal you've completed it. Enable auto-merge if
the PR is ready to merge.
- [ ] If the pull request requires a cryptography review (e.g.
cryptographic algorithm implementations) I have added the 'crypto' tag.
- [ ] I have reviewed my diff in github, line by line and removed
unexpected formatting changes, testing logs, or commented-out code.
- [ ] Every change is related to the PR description.
- [ ] I have
[linked](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue)
this pull request to relevant issues (if any exist).
---
 .circleci/config.yml                          | 86 ++++++++++++------
 build-system/scripts/deploy_dockerhub         | 15 ++--
 yarn-project/bootstrap.sh                     |  2 +-
 .../canary/scripts/docker-compose-browser.yml |  2 +-
 .../canary/scripts/docker-compose.yml         |  2 +-
 yarn-project/canary/scripts/run_tests         |  2 +-
 .../canary/scripts/update_packages.sh         | 19 ++--
 yarn-project/deploy_dockerhub.sh              | 17 ++--
 yarn-project/deploy_npm.sh                    | 87 ++++++++++---------
 .../end-to-end/scripts/setup_canary.sh        | 29 ++-----
 10 files changed, 144 insertions(+), 117 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index ad12ed2b33c..bc968b697fb 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1024,6 +1024,15 @@ jobs:
           name: "Noop"
           command: echo Noop
 
+  canary-end:
+    docker:
+      - image: cimg/base:2023.09
+    resource_class: small
+    steps:
+      - run:
+          name: "Noop"
+          command: echo Noop
+
   bench-summary:
     machine:
       image: ubuntu-2204:2023.07.2
@@ -1034,6 +1043,19 @@ jobs:
           name: "Assemble benchmark summary from uploaded logs"
           command: ./scripts/ci/assemble_e2e_benchmark.sh
 
+  deploy-npm-canary:
+    # Deploys next version under 'canary' dist tag.
+    machine:
+      image: ubuntu-2204:2023.07.2
+    resource_class: medium
+    steps:
+      - *checkout
+      - *setup_env
+      # Aztec.js and dependencies
+      - run:
+          name: "yarn-project"
+          command: yarn-project/deploy_npm.sh canary
+
   deploy-npm:
     machine:
       image: ubuntu-2204:2023.07.2
@@ -1044,7 +1066,18 @@ jobs:
       # Aztec.js and dependencies
       - run:
           name: "yarn-project"
-          command: yarn-project/deploy_npm.sh
+          command: yarn-project/deploy_npm.sh latest
+
+  deploy-dockerhub-canary:
+    machine:
+      image: ubuntu-2204:2023.07.2
+    resource_class: medium
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Deploy to dockerhub"
+          command: yarn-project/deploy_dockerhub.sh canary
 
   deploy-ecr:
     machine:
@@ -1068,15 +1101,6 @@ jobs:
           name: "Deploy to dockerhub"
           command: yarn-project/deploy_dockerhub.sh
 
-  deploy-end:
-    docker:
-      - image: cimg/base:2023.09
-    resource_class: small
-    steps:
-      - run:
-          name: "Noop"
-          command: echo Noop
-
   build-deployment-canary:
     machine:
       image: ubuntu-2204:2023.07.2
@@ -1088,7 +1112,7 @@ jobs:
           name: "Build"
           command: build canary true
 
-  run-deployment-canary-uniswap:
+  canary-uniswap-test:
     machine:
       image: ubuntu-2204:2023.07.2
     resource_class: large
@@ -1099,7 +1123,7 @@ jobs:
           name: "Test"
           command: run_script canary ./scripts/run_tests ./src/uniswap_trade_on_l1_from_l2.test.ts canary ./scripts/docker-compose.yml
 
-  run-deployment-canary-browser:
+  canary-browser-test:
     machine:
       image: ubuntu-2204:2023.07.2
     resource_class: large
@@ -1110,7 +1134,7 @@ jobs:
           name: "Test"
           command: run_script canary ./scripts/run_tests ./src/aztec_js_browser.test.ts canary ./scripts/docker-compose-browser.yml
 
-  run-deployment-canary-cli:
+  canary-cli-test:
     machine:
       image: ubuntu-2204:2023.07.2
     resource_class: large
@@ -1408,12 +1432,12 @@ workflows:
             - bench-process-history
           <<: *defaults
 
-      # Deployment and Canary tests
-      - deploy-dockerhub:
+        # Deploy under canary tag
+      - deploy-npm-canary:
           requires:
             - e2e-end
           <<: *deploy_defaults
-      - deploy-npm:
+      - deploy-dockerhub-canary:
           requires:
             - e2e-end
           <<: *deploy_defaults
@@ -1422,28 +1446,40 @@ workflows:
             - e2e-end
           <<: *deploy_defaults
 
-      - deploy-end:
+      - build-deployment-canary:
           requires:
-            - deploy-dockerhub
-            - deploy-npm
+            - deploy-npm-canary
+            - deploy-dockerhub-canary
           <<: *deploy_defaults
 
-      - build-deployment-canary:
+      # Run canary tests
+      - canary-uniswap-test:
           requires:
-            - deploy-end
+            - build-deployment-canary
           <<: *deploy_defaults
 
-      - run-deployment-canary-uniswap:
+      - canary-browser-test:
           requires:
             - build-deployment-canary
           <<: *deploy_defaults
 
-      - run-deployment-canary-browser:
+      - canary-cli-test:
           requires:
             - build-deployment-canary
           <<: *deploy_defaults
 
-      - run-deployment-canary-cli:
+      - canary-end:
           requires:
-            - build-deployment-canary
+            - canary-uniswap-test
+            - canary-browser-test
+            - canary-cli-test
+
+        # Production deployment
+      - deploy-dockerhub:
+          requires:
+            - canary-end
+          <<: *deploy_defaults
+      - deploy-npm:
+          requires:
+            - canary-end
           <<: *deploy_defaults
diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub
index 21d6a0e6406..e5782d970ba 100755
--- a/build-system/scripts/deploy_dockerhub
+++ b/build-system/scripts/deploy_dockerhub
@@ -9,6 +9,7 @@ fi
 
 REPOSITORY=$1
 ARCH=${2:-""}
+DIST_TAG=${3:-"latest"}
 
 echo "Repo: $REPOSITORY"
 echo "Arch: $ARCH"
@@ -18,10 +19,10 @@ USERNAME="aztecprotocolci"
 
 IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY)
 
-IMAGE_LATEST_URI=$ACCOUNT/$REPOSITORY:latest
+IMAGE_DIST_URI=$ACCOUNT/$REPOSITORY:$DIST_TAG
 if [[ -n "$ARCH" ]]; then
   IMAGE_COMMIT_URI=$IMAGE_COMMIT_URI-$ARCH
-  IMAGE_LATEST_URI=$IMAGE_LATEST_URI-$ARCH
+  IMAGE_DIST_URI=$IMAGE_DIST_URI-$ARCH
 fi
 
 COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY true)
@@ -49,11 +50,11 @@ echo "Tagging $IMAGE_COMMIT_URI as $IMAGE_DEPLOY_URI"
 # Tag with commit tag
 docker_or_dryrun tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_URI
 
-echo "Tagging $IMAGE_COMMIT_URI as $IMAGE_LATEST_URI"
-# Tag with :latest
-docker_or_dryrun tag $IMAGE_COMMIT_URI $IMAGE_LATEST_URI
+echo "Tagging $IMAGE_COMMIT_URI as $IMAGE_DIST_URI"
+# Tag with :$DIST_TAG (latest or canary)
+docker_or_dryrun tag $IMAGE_COMMIT_URI $IMAGE_DIST_URI
 
 # Push tagged image to dockerhub.
 docker_or_dryrun push $IMAGE_DEPLOY_URI
-# Push :latest image to dockerhub
-docker_or_dryrun push $IMAGE_LATEST_URI
+# Push :latest or :canary image to dockerhub
+docker_or_dryrun push $IMAGE_DIST_URI
diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh
index 2120e9d3300..a5fff3dedbe 100755
--- a/yarn-project/bootstrap.sh
+++ b/yarn-project/bootstrap.sh
@@ -6,7 +6,7 @@ cd "$(dirname "$0")"
 
 if [ "$(uname)" = "Darwin" ]; then
   # works around https://github.com/AztecProtocol/aztec3-packages/issues/158
-  echo "Note: not sourcing nvm on Mac, see github #158"
+    echo "Note: not sourcing nvm on Mac, see github #158"
 else
   \. ~/.nvm/nvm.sh
 fi
diff --git a/yarn-project/canary/scripts/docker-compose-browser.yml b/yarn-project/canary/scripts/docker-compose-browser.yml
index 77279f629c0..6e9429371ab 100644
--- a/yarn-project/canary/scripts/docker-compose-browser.yml
+++ b/yarn-project/canary/scripts/docker-compose-browser.yml
@@ -13,7 +13,7 @@ services:
       - '8545:8545'
 
   sandbox:
-    image: aztecprotocol/aztec-sandbox:latest
+    image: aztecprotocol/aztec-sandbox:canary
     environment:
       DEBUG: 'aztec:*'
       ETHEREUM_HOST: http://fork:8545
diff --git a/yarn-project/canary/scripts/docker-compose.yml b/yarn-project/canary/scripts/docker-compose.yml
index 2a06ec65107..06a2c094a19 100644
--- a/yarn-project/canary/scripts/docker-compose.yml
+++ b/yarn-project/canary/scripts/docker-compose.yml
@@ -13,7 +13,7 @@ services:
       - '8545:8545'
 
   sandbox:
-    image: aztecprotocol/aztec-sandbox:latest
+    image: aztecprotocol/aztec-sandbox:canary
     environment:
       DEBUG: 'aztec:*'
       ETHEREUM_HOST: http://fork:8545
diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests
index 959822fe3ef..5ed4d4609c1 100755
--- a/yarn-project/canary/scripts/run_tests
+++ b/yarn-project/canary/scripts/run_tests
@@ -9,7 +9,7 @@ export IMAGE=${2:-canary}
 export COMPOSE_FILE=${3:-./scripts/docker-compose.yml}
 :
 # if test name ends with uniswap_trade_on_l1_from_l2.test.ts, use the forked mainnet
-if [[ "$TEST" == *"uniswap_trade_on_l1_from_l2.test.ts" ]]; then
+if [[ "$TEST" == *"uniswap_trade_on_l1_from_l2"* ]]; then
   export FORK_URL=https://mainnet.infura.io/v3/9928b52099854248b3a096be07a6b23c
   export FORK_BLOCK_NUMBER=17514288
 fi
diff --git a/yarn-project/canary/scripts/update_packages.sh b/yarn-project/canary/scripts/update_packages.sh
index dac457c7248..ce020291a73 100755
--- a/yarn-project/canary/scripts/update_packages.sh
+++ b/yarn-project/canary/scripts/update_packages.sh
@@ -1,23 +1,18 @@
+#!/bin/bash
 set -eu
 
-COMMIT_TAG=$1
+DIST_TAG=$1
 
-if [ -z "$COMMIT_TAG" ]; then
-  echo "No commit tag provided."
+if [ -z "$DIST_TAG" ]; then
+  echo "No dist tag provided."
   exit 0
 fi
 
-VERSION=$(npx semver $COMMIT_TAG)
-if [ -z "$VERSION" ]; then
-  echo "$COMMIT_TAG is not a semantic version."
-  exit 1
-fi
-
-echo "Updating Aztec dependencies to version $VERSION"
+echo "Updating Aztec dependencies to tag $DIST_TAG"
 
 TMP=$(mktemp)
 for PKG in $(jq --raw-output ".dependencies | keys[] | select(contains(\"@aztec/\") and (. != \"@aztec/end-to-end\"))" package.json); do
-  jq --arg v $VERSION ".dependencies[\"$PKG\"] = \$v" package.json > $TMP && mv $TMP package.json
+  jq --arg v $DIST_TAG ".dependencies[\"$PKG\"] = \$v" package.json >$TMP && mv $TMP package.json
 done
 
-jq ".references = []" tsconfig.json > $TMP && mv $TMP tsconfig.json
+jq ".references = []" tsconfig.json >$TMP && mv $TMP tsconfig.json
diff --git a/yarn-project/deploy_dockerhub.sh b/yarn-project/deploy_dockerhub.sh
index 5e7d83dd891..9f5e483dbec 100755
--- a/yarn-project/deploy_dockerhub.sh
+++ b/yarn-project/deploy_dockerhub.sh
@@ -1,15 +1,16 @@
 #!/bin/bash
 
+DIST_TAG=${1:-"latest"}
+
 extract_repo yarn-project /usr/src project
 PROJECT_ROOT=$(pwd)/project/src/
 
-for REPOSITORY in "pxe" "aztec-sandbox"
-do
-    echo "Deploying $REPOSITORY"
-    RELATIVE_PROJECT_DIR=$(query_manifest relativeProjectDir $REPOSITORY)
-    cd "$PROJECT_ROOT/$RELATIVE_PROJECT_DIR"
+for REPOSITORY in "pxe" "aztec-sandbox"; do
+  echo "Deploying $REPOSITORY $TAG"
+  RELATIVE_PROJECT_DIR=$(query_manifest relativeProjectDir $REPOSITORY)
+  cd "$PROJECT_ROOT/$RELATIVE_PROJECT_DIR"
 
-    deploy_dockerhub $REPOSITORY x86_64
-    deploy_dockerhub $REPOSITORY arm64
-    create_dockerhub_manifest $REPOSITORY x86_64,arm64
+  deploy_dockerhub $REPOSITORY x86_64 $DIST_TAG
+  deploy_dockerhub $REPOSITORY arm64 $DIST_TAG
+  create_dockerhub_manifest $REPOSITORY x86_64,arm64 $DIST_TAG
 done
diff --git a/yarn-project/deploy_npm.sh b/yarn-project/deploy_npm.sh
index 77a370faea7..f391b489176 100755
--- a/yarn-project/deploy_npm.sh
+++ b/yarn-project/deploy_npm.sh
@@ -5,63 +5,72 @@ set -eu
 extract_repo yarn-project /usr/src project
 cd project/src/yarn-project
 
-echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > .npmrc
+echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >.npmrc
 # also copy npcrc into the l1-contracts directory
 cp .npmrc ../l1-contracts
 
+# This is to be used with the 'canary' tag for testing, and then 'latest' for making it public
+DIST_TAG=${1:-"latest"}
+
 function deploy_package() {
-    REPOSITORY=$1
-    cd $REPOSITORY
+  REPOSITORY=$1
+  cd $REPOSITORY
 
-    VERSION=$(extract_tag_version $REPOSITORY false)
-    echo "Deploying $REPOSITORY $VERSION"
+  PACKAGE_NAME=$(jq -r '.name' package.json)
+  VERSION=$(extract_tag_version $REPOSITORY false)
+  echo "Deploying $REPOSITORY $VERSION $DIST_TAG"
 
-    # If the commit tag itself has a dist-tag (e.g. v2.1.0-testnet.123), extract the dist-tag.
-    TAG=$(echo "$VERSION" | grep -oP ".*-\K(.*)(?=\.\d+)" || true)
-    TAG_ARG=""
-    if [ -n "$TAG" ]; then
-        TAG_ARG="--tag $TAG"
-    fi
+  if [ -n "$DIST_TAG" ]; then
+    TAG_ARG="--tag $DIST_TAG"
+  fi
 
-    PUBLISHED_VERSION=$(npm show . version ${TAG_ARG:-} 2> /dev/null) || true
-    HIGHER_VERSION=$(npx semver ${VERSION} ${PUBLISHED_VERSION} | tail -1)
+  PUBLISHED_VERSION=$(npm show . version ${TAG_ARG:-} 2>/dev/null) || true
+  HIGHER_VERSION=$(npx semver ${VERSION} ${PUBLISHED_VERSION} | tail -1)
 
-    # If there is already a published package equal to given version, assume this is a re-run of a deploy, and early out.
-    if [ "$VERSION" == "$PUBLISHED_VERSION" ]; then
-        echo "Tagged version $VERSION is equal to published version $PUBLISHED_VERSION. Skipping publish."
-        exit 0
-    fi
+  # Check if there is already a published package equal to given version, assume this is a re-run of a deploy
+  if [ "$VERSION" == "$PUBLISHED_VERSION" ]; then
+    echo "Tagged ${DIST_TAG:+ $DIST_TAG}version $VERSION is equal to published ${DIST_TAG:+ $DIST_TAG}version $PUBLISHED_VERSION."
+    echo "Skipping publish."
+    exit 0
+  fi
 
-    # If the published version is > the given version, something's gone wrong.
-    if [ "$VERSION" != "$HIGHER_VERSION" ]; then
-        echo "Tagged version $VERSION is lower than published version $PUBLISHED_VERSION."
-        exit 1
-    fi
+  # If the published version is > the given version, something's gone wrong.
+  if [ "$VERSION" != "$HIGHER_VERSION" ]; then
+    echo "Tagged version $VERSION is lower than published version $PUBLISHED_VERSION."
+    exit 1
+  fi
 
-    # Update the package version in package.json.
-    TMP=$(mktemp)
-    jq --arg v $VERSION '.version = $v' package.json > $TMP && mv $TMP package.json
+  # Update the package version in package.json.
+  TMP=$(mktemp)
+  jq --arg v $VERSION '.version = $v' package.json >$TMP && mv $TMP package.json
 
-    if [ -z "${STANDALONE:-}" ]; then
+  if [ -z "${STANDALONE:-}" ]; then
     # Update each dependent @aztec package version in package.json.
     for PKG in $(jq --raw-output ".dependencies | keys[] | select(contains(\"@aztec/\"))" package.json); do
-        jq --arg v $VERSION ".dependencies[\"$PKG\"] = \$v" package.json > $TMP && mv $TMP package.json
+      jq --arg v $VERSION ".dependencies[\"$PKG\"] = \$v" package.json >$TMP && mv $TMP package.json
     done
-    fi
+  fi
 
-    # Publish
-    if [ -n "${COMMIT_TAG:-}" ] ; then 
-        npm publish $TAG_ARG --access public
+  # Publish
+  if [ -n "${COMMIT_TAG:-}" ]; then
+    # Check if version exists
+    if npm view "$PACKAGE_NAME@$VERSION" version >/dev/null 2>&1; then
+      # Tag the existing version
+      npm dist-tag add $PACKAGE_NAME@$VERSION $DIST_TAG
     else
-        npm publish --dry-run $TAG_ARG --access public
+      # Publish new verison
+      npm publish $TAG_ARG --access public
     fi
+  else
+    npm publish --dry-run $TAG_ARG --access public
+  fi
 
-    # Back to root
-    if [ "$REPOSITORY" == "../l1-contracts" ]; then
-        cd ../yarn-project
-    else
-        cd ..
-    fi
+  # Back to root
+  if [ "$REPOSITORY" == "../l1-contracts" ]; then
+    cd ../yarn-project
+  else
+    cd ..
+  fi
 }
 
 deploy_package foundation
diff --git a/yarn-project/end-to-end/scripts/setup_canary.sh b/yarn-project/end-to-end/scripts/setup_canary.sh
index e7ea1020221..db900ca2843 100755
--- a/yarn-project/end-to-end/scripts/setup_canary.sh
+++ b/yarn-project/end-to-end/scripts/setup_canary.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -eu
 
-COMMIT_TAG=$1
+DIST_TAG=$1
 TARGET_PKGS_FILE=$2
 
 # Check if file exists and read it into an array
@@ -15,26 +15,11 @@ else
   echo "File $TARGET_PKGS_FILE does not exist."
 fi
 
-if [ -z "$COMMIT_TAG" ]; then
-  echo "No commit tag provided."
+if [ -z "$DIST_TAG" ]; then
+  echo "No dist tag provided."
   exit 0
 fi
 
-set +e  # Temporarily disable exit on error
-VERSION=$(npx semver $COMMIT_TAG)
-RESULT=$?  # Capture the exit status of the last command
-set -e  # Re-enable exit on error
-
-if [ $RESULT -ne 0 ]; then
-  echo "Error when running 'npx semver' with commit tag: $COMMIT_TAG"
-  exit 1
-fi
-
-if [ -z "$VERSION" ]; then
-  echo "$COMMIT_TAG is not a semantic version."
-  exit 1
-fi
-
 echo "Removing all files & folders that aren't needed for canary tests"
 TARGET_DIR="./src"
 cd "$TARGET_DIR"
@@ -48,11 +33,11 @@ for item in $(ls -A); do
 done
 cd ..
 
-echo "Updating external Aztec dependencies to version $VERSION"
+echo "Updating external Aztec dependencies to tag $DIST_TAG"
 JSON_TARGET_PKGS=$(printf '%s\n' "${TARGET_PKGS[@]}" | jq -R -s -c 'split("\n") | map(select(. != ""))')
 
 TMP=$(mktemp)
-jq --arg v $VERSION --argjson target_pkgs "$JSON_TARGET_PKGS" '
+jq --arg v $DIST_TAG --argjson target_pkgs "$JSON_TARGET_PKGS" '
 .dependencies |= with_entries(
   select(
     (.key | startswith("@aztec")) as $isAztec |
@@ -67,6 +52,6 @@ jq --arg v $VERSION --argjson target_pkgs "$JSON_TARGET_PKGS" '
   else
     .
   end
-)' package.json > $TMP && mv $TMP package.json
+)' package.json >$TMP && mv $TMP package.json
 
-jq ".references = []" tsconfig.json > $TMP && mv $TMP tsconfig.json
+jq ".references = []" tsconfig.json >$TMP && mv $TMP tsconfig.json