diff --git a/.circleci/config.yml b/.circleci/config.yml index ff3c13631a6..6284197e53e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -465,6 +465,17 @@ jobs: name: "Build and test" command: build aztec-node | add_timestamps + mainnet-fork: + machine: + image: ubuntu-2204:2023.07.2 + resource_class: large + steps: + - *checkout + - *setup_env + - run: + name: "Build" + command: build mainnet-fork | add_timestamps + aztec-faucet: machine: image: ubuntu-2204:2023.07.2 @@ -1222,6 +1233,8 @@ workflows: - l1-contracts: *defaults - noir-contracts-build: *defaults + - mainnet-fork: *defaults + # Yarn Project - yarn-project-base: requires: diff --git a/build_manifest.yml b/build_manifest.yml index c3d413b1552..031e7439662 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -174,6 +174,10 @@ p2p-bootstrap: dependencies: - yarn-project +mainnet-fork: + buildDir: iac/mainnet-fork + projectDir: iac/mainnet-fork + docs: buildDir: . dockerfile: docs/Dockerfile diff --git a/iac/mainnet-fork/Dockerfile b/iac/mainnet-fork/Dockerfile new file mode 100644 index 00000000000..c0240ccadc8 --- /dev/null +++ b/iac/mainnet-fork/Dockerfile @@ -0,0 +1,18 @@ +FROM ubuntu:focal + +# Install nginx +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections +RUN apt-get update && apt install -y git curl nginx + +# Copy nginx config +COPY . . +COPY nginx/gateway.conf /etc/nginx/gateway.conf +COPY nginx/nginx.conf /etc/nginx/nginx.conf + +# Install foundry +RUN ./scripts/install_foundry.sh +ENV PATH="./foundry/bin:${PATH}" + +# Run anvil and nginx +EXPOSE 80 +ENTRYPOINT ["sh", "-c", "./scripts/run_nginx_anvil.sh"] \ No newline at end of file diff --git a/iac/mainnet-fork/nginx/gateway.conf b/iac/mainnet-fork/nginx/gateway.conf new file mode 100644 index 00000000000..74f889b9eff --- /dev/null +++ b/iac/mainnet-fork/nginx/gateway.conf @@ -0,0 +1,14 @@ +server { + listen 80 default_server; + listen 8545; + + location = /{{API_KEY}} { + proxy_pass http://0.0.0.0:8544; + rewrite ^/{{API_KEY}}(.*) /$1 break; + } + + # Error responses + error_page 404 = @400; # Treat invalid paths as bad requests + proxy_intercept_errors on; # Do not send backend errors to client + default_type application/json; # If no content-type, assume JSON +} \ No newline at end of file diff --git a/iac/mainnet-fork/nginx/nginx.conf b/iac/mainnet-fork/nginx/nginx.conf new file mode 100644 index 00000000000..0deef80724e --- /dev/null +++ b/iac/mainnet-fork/nginx/nginx.conf @@ -0,0 +1,53 @@ + +events { + worker_connections 768; + # multi_accept on; +} + +http { + + ## + # Basic Settings + ## + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # SSL Settings + ## + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE + ssl_prefer_server_ciphers on; + + ## + # Logging Settings + ## + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + ## + # Gzip Settings + ## + + gzip on; + + # gzip_vary on; + # gzip_proxied any; + # gzip_comp_level 6; + # gzip_buffers 16 8k; + # gzip_http_version 1.1; + # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + + + include /etc/nginx/gateway.conf; + include /etc/nginx/conf.d/*.conf; +} + diff --git a/iac/mainnet-fork/scripts/install_foundry.sh b/iac/mainnet-fork/scripts/install_foundry.sh new file mode 100755 index 00000000000..3842f6a4318 --- /dev/null +++ b/iac/mainnet-fork/scripts/install_foundry.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu + +export FOUNDRY_DIR="$PWD/.foundry" +FOUNDRY_BIN_DIR="$FOUNDRY_DIR/bin" +BIN_URL="https://raw.githubusercontent.com/foundry-rs/foundry/master/foundryup/foundryup" +BIN_PATH="$FOUNDRY_BIN_DIR/foundryup" +FOUNDRY_MAN_DIR="$FOUNDRY_DIR/share/man/man1" + +# Clean +rm -rf $FOUNDRY_DIR + +# Install foundryup. +mkdir -p $FOUNDRY_BIN_DIR +mkdir -p $FOUNDRY_MAN_DIR +curl -# -L $BIN_URL -o $BIN_PATH +chmod +x $BIN_PATH +export PATH=$FOUNDRY_BIN_DIR:$PATH + +# Use version. +foundryup \ No newline at end of file diff --git a/iac/mainnet-fork/scripts/run_nginx_anvil.sh b/iac/mainnet-fork/scripts/run_nginx_anvil.sh new file mode 100755 index 00000000000..4b701f13256 --- /dev/null +++ b/iac/mainnet-fork/scripts/run_nginx_anvil.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -eum pipefail + +# Replace API_KEY in nginx config +echo "Replacing api key with $API_KEY in nginx config..." +sed -i 's/{{API_KEY}}/'$API_KEY'/' /etc/nginx/gateway.conf + +# Run nginx and anvil alongside each other +trap 'kill $(jobs -p)' SIGTERM + +# Anvil defaults - Nginx assumes these values to be as they are +HOST="0.0.0.0" +PORT=8544 +ETHEREUM_HOST=$HOST:$PORT + +# Data directory for anvil state +mkdir -p /data + +# Run anvil silently +.foundry/bin/anvil --silent --host $HOST -p $PORT -m "$MNEMONIC" -f=https://mainnet.infura.io/v3/$INFURA_API_KEY --chain-id=$CHAIN_ID --fork-block-number=15918000 --block-base-fee-per-gas=10 -s=$SNAPSHOT_FREQUENCY --state=./data/state --balance=1000000000000000000 >/dev/null & + +echo "Waiting for ethereum host at $ETHEREUM_HOST..." +while ! curl -s $ETHEREUM_HOST >/dev/null; do sleep 1; done + +echo "Starting nginx..." +nginx & +wait diff --git a/iac/mainnet-fork/scripts/wait_for_fork b/iac/mainnet-fork/scripts/wait_for_fork new file mode 100755 index 00000000000..4d990e30f88 --- /dev/null +++ b/iac/mainnet-fork/scripts/wait_for_fork @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +# When destroying and applying mainnet fork terraform, it may not be +# ready for a while, as it must register with DNS etc. +# This script waits on a healthy status from the fork - a valid response to the chainid request +# We retry every 20 seconds, and wait for a total of 5 minutes (15 times) + +export ETHEREUM_HOST="https://aztec-mainnet-fork.aztec.network:8545/$FORK_API_KEY" + +curl -H "Content-Type: application/json" -X POST --data '{"method":"eth_chainId","params":[],"id":33,"jsonrpc":"2.0"}' \ + --connect-timeout 30 \ + --retry 15 \ + --retry-delay 20 \ + $ETHEREUM_HOST diff --git a/iac/mainnet-fork/terraform/main.tf b/iac/mainnet-fork/terraform/main.tf new file mode 100644 index 00000000000..beb2ab35357 --- /dev/null +++ b/iac/mainnet-fork/terraform/main.tf @@ -0,0 +1,276 @@ +terraform { + backend "s3" { + bucket = "aztec-terraform" + key = "aztec-network/mainnet-fork" + region = "eu-west-2" + } + required_providers { + aws = { + source = "hashicorp/aws" + version = "3.74.2" + } + } +} + +data "terraform_remote_state" "setup_iac" { + backend = "s3" + config = { + bucket = "aztec-terraform" + key = "setup/setup-iac" + region = "eu-west-2" + } +} + +data "terraform_remote_state" "aztec-network_iac" { + backend = "s3" + config = { + bucket = "aztec-terraform" + key = "aztec-network/iac" + region = "eu-west-2" + } +} + +data "terraform_remote_state" "aztec2_iac" { + backend = "s3" + config = { + bucket = "aztec-terraform" + key = "aztec2/iac" + region = "eu-west-2" + } +} + + +data "aws_alb" "aztec-network_alb" { + arn = data.terraform_remote_state.aztec2_iac.outputs.alb_arn +} + +provider "aws" { + profile = "default" + region = "eu-west-2" +} + +resource "aws_service_discovery_service" "aztec_mainnet_fork" { + name = "aztec-network-mainnet-fork" + + health_check_custom_config { + failure_threshold = 1 + } + + dns_config { + namespace_id = data.terraform_remote_state.setup_iac.outputs.local_service_discovery_id + + dns_records { + ttl = 60 + type = "A" + } + + dns_records { + ttl = 60 + type = "SRV" + } + + routing_policy = "MULTIVALUE" + } +} + +# EFS filesystem for mainnet fork +resource "aws_efs_file_system" "aztec_mainnet_fork_data_store" { + creation_token = "aztec-network-mainnet-fork-data" + + tags = { + Name = "aztec-network-mainnet-fork-data" + } + + lifecycle_policy { + transition_to_ia = "AFTER_30_DAYS" + } +} + +resource "aws_efs_mount_target" "aztec_fork_private_az1" { + file_system_id = aws_efs_file_system.aztec_mainnet_fork_data_store.id + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id + security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] +} + +resource "aws_efs_mount_target" "aztec_fork_private_az2" { + file_system_id = aws_efs_file_system.aztec_mainnet_fork_data_store.id + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id + security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] +} + +# Define deployment task and service +resource "aws_ecs_task_definition" "aztec_mainnet_fork" { + family = "aztec-network-mainnet-fork" + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = "2048" + memory = "4096" + execution_role_arn = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn + + volume { + name = "efs-data-store" + efs_volume_configuration { + file_system_id = aws_efs_file_system.aztec_mainnet_fork_data_store.id + } + } + + container_definitions = <