From cc515dabf981d4fd0938f8650c200b54288586a2 Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 14 Dec 2023 13:47:58 +0000 Subject: [PATCH] fix(ci): redeploy triggers (#3677) - use `deploy_service` to restart AWS services - check if l1-contracts were redeployed to reset node file systems - redeploy l1-contracts --- .circleci/config.yml | 32 ++++++++++---- build-system/scripts/deploy_service | 24 ++++++++-- build-system/scripts/deploy_terraform | 2 +- .../scripts/deploy_terraform_services | 44 +++++++++++++++++++ build-system/scripts/setup_env | 10 +++-- build-system/scripts/should_deploy | 2 +- build_manifest.yml | 12 +++++ l1-contracts/REDEPLOY | 2 +- l1-contracts/scripts/ci_deploy_contracts.sh | 18 ++++++-- yarn-project/aztec-faucet/terraform/main.tf | 8 ++-- yarn-project/aztec-node/terraform/main.tf | 2 +- yarn-project/p2p-bootstrap/terraform/main.tf | 40 ++++++++--------- 12 files changed, 148 insertions(+), 48 deletions(-) create mode 100755 build-system/scripts/deploy_terraform_services diff --git a/.circleci/config.yml b/.circleci/config.yml index 2cb1229ab39..5bc9e5b19e8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -945,7 +945,7 @@ jobs: name: "Deploy mainnet fork" command: | should_deploy || exit 0 - deploy mainnet-fork + deploy_terraform_services iac/mainnet-fork - run: name: "Deploy L1 contracts to mainnet fork" working_directory: l1-contracts @@ -953,19 +953,35 @@ jobs: should_deploy || exit 0 ./scripts/ci_deploy_contracts.sh - run: - name: "Deploy devnet to AWS" + name: "Deploy P2P bootstrap servers to AWS" command: | should_deploy 0 || exit 0 - export TF_VAR_FAUCET_PRIVATE_KEY=$FAUCET_PRIVATE_KEY - export TF_VAR_BOOTNODE_1_PEER_ID=$BOOTNODE_1_PEER_ID - export TF_VAR_BOOTNODE_2_PEER_ID=$BOOTNODE_2_PEER_ID + # Export variables for Terraform. export TF_VAR_BOOTNODE_1_PRIVATE_KEY=$BOOTNODE_1_PRIVATE_KEY export TF_VAR_BOOTNODE_2_PRIVATE_KEY=$BOOTNODE_2_PRIVATE_KEY + deploy_terraform_services yarn-project/p2p-bootstrap aztec-sandbox + - run: + name: "Deploy Aztec Nodes to AWS" + command: | + should_deploy 0 || exit 0 + export TF_VAR_BOOTNODE_1_PEER_ID=$BOOTNODE_1_PEER_ID + export TF_VAR_BOOTNODE_2_PEER_ID=$BOOTNODE_2_PEER_ID export TF_VAR_SEQ_1_PUBLISHER_PRIVATE_KEY=$SEQ_1_PUBLISHER_PRIVATE_KEY export TF_VAR_SEQ_2_PUBLISHER_PRIVATE_KEY=$SEQ_2_PUBLISHER_PRIVATE_KEY - deploy_terraform p2p-bootstrap yarn-project/p2p-bootstrap/terraform - deploy_terraform aztec-node yarn-project/aztec-node/terraform - deploy_terraform aztec-faucet yarn-project/aztec-faucet/terraform + export TF_VAR_NODE_1_PRIVATE_KEY=$NODE_1_PRIVATE_KEY + export TF_VAR_NODE_2_PRIVATE_KEY=$NODE_2_PRIVATE_KEY + # Check if l1-contracts have changed + if $CONTRACTS_DEPLOYED -eq 1; then + deploy_terraform_services yarn-project/aztec-node aztec-sandbox aztec-node aws_efs_file_system.node_data_store + else + deploy_terraform_services yarn-project/aztec-node aztec-sandbox + fi + - run: + name: "Deploy Aztec Faucet to AWS" + command: | + should_deploy 0 || exit 0 + export TF_VAR_FAUCET_PRIVATE_KEY=$FAUCET_PRIVATE_KEY + deploy_terraform_services yarn-project/aztec-faucet aztec-sandbox # Repeatable config for defining the workflow below. defaults: &defaults diff --git a/build-system/scripts/deploy_service b/build-system/scripts/deploy_service index 83c45128141..80fad879ad4 100755 --- a/build-system/scripts/deploy_service +++ b/build-system/scripts/deploy_service @@ -2,8 +2,24 @@ [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu -# Redeploy service with latest image. +# Redeploy services with the latest image that match $DEPLOY_TAG followed by $SERVICE_NAME. SERVICE_NAME=$1 -if aws ecs list-services --region $ECR_DEPLOY_REGION --cluster setup | grep "/$SERVICE_NAME\"" > /dev/null; then - aws ecs update-service --region $ECR_DEPLOY_REGION --cluster setup --service $SERVICE_NAME --force-new-deployment -fi +PATTERN="$DEPLOY_TAG.*$SERVICE_NAME.*" + +# Fetch list of services +SERVICES=$(aws ecs list-services --region $ECR_DEPLOY_REGION --cluster setup | grep -Eo "arn:aws:ecs:[^:]+:[^:]+:service/[^/]+/$PATTERN" || true) + +echo "Services to redeploy:" +echo "$SERVICES" + +# Loop through and update each matching service. +for SERVICE_ARN in $SERVICES; do + # Extract the actual service name from ARN + ACTUAL_SERVICE_NAME=$(echo "$SERVICE_ARN" | awk -F/ '{print $NF}') + + if [ "$DRY_DEPLOY" -eq 1 ]; then + echo "DRY_DEPLOY: aws ecs update-service --region $ECR_DEPLOY_REGION --cluster setup --service $ACTUAL_SERVICE_NAME --force-new-deployment" + else + aws ecs update-service --region $ECR_DEPLOY_REGION --cluster setup --service $ACTUAL_SERVICE_NAME --force-new-deployment + fi +done diff --git a/build-system/scripts/deploy_terraform b/build-system/scripts/deploy_terraform index f5ff4dc810b..1575b7f46f4 100755 --- a/build-system/scripts/deploy_terraform +++ b/build-system/scripts/deploy_terraform @@ -44,7 +44,7 @@ for RESOURCE in $TO_TAINT; do done if [ "$DRY_DEPLOY" -eq 1 ]; then - terraform plan -input=false -auto-approve + terraform plan -input=false else terraform apply -input=false -auto-approve fi diff --git a/build-system/scripts/deploy_terraform_services b/build-system/scripts/deploy_terraform_services new file mode 100755 index 00000000000..879ca6a5290 --- /dev/null +++ b/build-system/scripts/deploy_terraform_services @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +# This script deploys a terraform project and restarts related services. + +# The terraform project directory. +PROJECT_DIR=$1 +# Extract project name fromm the directory, e.g. yarn-project/aztec-node -> aztec-node +PROJECT_NAME=$(basename $PROJECT_DIR) + +# The repository to check for changes. Defaults to the project name +# but can be different for projects that e.g. use the sandbox image. +CHECK_REBUILD_REPOSITORY=${2:-$PROJECT_NAME} + +# The services to restart. Defaults to the project name but can be different. +SERVICE_NAMES=${3:-$PROJECT_NAME} + +# The terraform resources to taint. Defaults to none. +TO_TAINT=${4:-} + +cd $PROJECT_DIR + +# Bail out if nothing changed. +CONTENT_HASH=$(calculate_content_hash $CHECK_REBUILD_REPOSITORY) +echo "Last successfully deployed commit: $CONTENT_HASH" +if check_rebuild cache-$CONTENT_HASH-$DEPLOY_TAG-deployed $CHECK_REBUILD_REPOSITORY; then + echo "No changes detected, skipping deployment." + exit 0 +fi + +deploy_terraform $PROJECT_NAME ./terraform/ "$TO_TAINT" + +# Restart services. +for SERVICE in $SERVICE_NAMES; do + deploy_service $SERVICE +done + +# Tag the image as deployed. +if [ "$DRY_DEPLOY" -eq 1 ]; then + echo "DRY_DEPLOY: tag_remote_image $CHECK_REBUILD_REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed" +else + retry tag_remote_image $CHECK_REBUILD_REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed +fi diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 3486e5f8653..f5db7733d93 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -34,7 +34,7 @@ echo "PULL_REQUEST=$PULL_REQUEST" # If the user has requested to perform a "dry deploy", we set the commit tag to fake version, and set DRY_DEPLOY to 1. if [[ "$COMMIT_MESSAGE" == *"[ci dry-deploy]"* ]]; then COMMIT_TAG=v999.999.999 - DRY_DEPLOY=1 + export DRY_DEPLOY=1 fi if should_deploy; then @@ -42,12 +42,14 @@ if should_deploy; then # Extract the deploy env from the commit tag, if it has one, e.g. testnet. # If we have one, we look something like v2.1.123-testnet.0. This is a "non production" release. if [[ "$COMMIT_TAG" == *"-"* ]]; then - # Strips the trailing '.XX' from the end of the commit tag + # Strips the trailing '.XX' from the end of the commit tag. TEMP=${COMMIT_TAG%.*} - # Strips the 'vX.Y.ZZZ-' from the front of the commit tag, leaving the e.g. 'testnet' + # Strips the 'vX.Y.ZZZ-' from the front of the commit tag, leaving the e.g. 'testnet'. DEPLOY_ENV=${TEMP##*-} - else + elif [ ! "$DRY_DEPLOY" -eq 1 ]; then DEPLOY_ENV=prod + else + DEPLOY_ENV=dev fi else # If we're on master, this is our devnet. diff --git a/build-system/scripts/should_deploy b/build-system/scripts/should_deploy index 283191f51f3..1595ba39ac9 100755 --- a/build-system/scripts/should_deploy +++ b/build-system/scripts/should_deploy @@ -3,7 +3,7 @@ # Right now, that's only if we're master. set -eu -if [ "$BRANCH" == "master" ]; then +if [ "$BRANCH" == "master" ] || [ "$DRY_DEPLOY" -eq 1 ]; then exit 0 else exit 1 diff --git a/build_manifest.yml b/build_manifest.yml index cea9275b042..7208deb365f 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -149,6 +149,18 @@ aztec-faucet: dependencies: - yarn-project-prod +aztec-node: + buildDir: yarn-project + projectDir: yarn-project/aztec-node + dependencies: + - yarn-project-prod + +p2p-bootstrap: + buildDir: yarn-project + projectDir: yarn-project/p2p-bootstrap + dependencies: + - yarn-project-prod + cli: buildDir: yarn-project projectDir: yarn-project/cli diff --git a/l1-contracts/REDEPLOY b/l1-contracts/REDEPLOY index fe5516129de..5407c1d3c07 100644 --- a/l1-contracts/REDEPLOY +++ b/l1-contracts/REDEPLOY @@ -1,2 +1,2 @@ # Append value to force redeploy -1 \ No newline at end of file +3 \ No newline at end of file diff --git a/l1-contracts/scripts/ci_deploy_contracts.sh b/l1-contracts/scripts/ci_deploy_contracts.sh index aaedb6f2f7d..15a0050d34a 100755 --- a/l1-contracts/scripts/ci_deploy_contracts.sh +++ b/l1-contracts/scripts/ci_deploy_contracts.sh @@ -11,6 +11,8 @@ echo "Last successfully published commit: $CONTENT_HASH" # Check if image hash has alredy been deployed. if check_rebuild "cache-$CONTENT_HASH-$DEPLOY_TAG-deployed" $REPOSITORY; then echo "No changes detected, no contract deploy necessary." + # Set global variable for redeployment of contracts + echo export CONTRACTS_DEPLOYED=0 >>$BASH_ENV exit 0 fi @@ -31,8 +33,16 @@ for KEY in ROLLUP_CONTRACT_ADDRESS REGISTRY_CONTRACT_ADDRESS INBOX_CONTRACT_ADDR export TF_VAR_$KEY=$VALUE done -# Write TF state variables -deploy_terraform l1-contracts ./terraform +if [ -n "${DRY_DEPLOY:-}" ]; then + echo "DRY_DEPLOY: deploy_terraform l1-contracts ./terraform" + echo "DRY_DEPLOY: tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed" +else + # Write TF state variables + deploy_terraform l1-contracts ./terraform -# Tag the image as deployed. -retry tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed + # Tag the image as deployed. + retry tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed +fi + +# Set global variable for redeployment of contracts +echo export CONTRACTS_DEPLOYED=1 >>$BASH_ENV diff --git a/yarn-project/aztec-faucet/terraform/main.tf b/yarn-project/aztec-faucet/terraform/main.tf index 68b0bd5e291..db33a2656c5 100644 --- a/yarn-project/aztec-faucet/terraform/main.tf +++ b/yarn-project/aztec-faucet/terraform/main.tf @@ -86,7 +86,7 @@ resource "aws_ecs_task_definition" "aztec-faucet" { container_definitions = <