Skip to content

Commit

Permalink
Merge branch 'master' into sync-noir
Browse files Browse the repository at this point in the history
* master:
  feat: verify public validation requests (#8150)
  feat: ultra keccak honk verifier (#8261)
  git subrepo push --branch=master noir-projects/aztec-nr
  git_subrepo.sh: Fix parent in .gitrepo file. [skip ci]
  chore: replace relative paths to noir-protocol-circuits
  git subrepo push --branch=master barretenberg
  chore(bb): use std::span for srs (#8371)
  feat: router contract (#8352)
  chore: Merge provernet to master (#8373)
  feat: Sync from noir (#8363)
  chore: Change efs volumes to use bursting throughput (#8370)
  chore: move spartan network tests to nightly (#8369)
  chore(ci): Test lowering of non-persistent ebs provisions (#8360)
  chore(bb): reinstate "chore: uncomment asserts in oink rec verifier"" (#8356)
  fix: Split stores per component and split merkle tree operations (#8299)
  fix: TXE logs in docker (#8365)
  feat: Liveness analysis for constants (#8294)
  chore(avm): remove some unused deps (#8366)
  • Loading branch information
TomAFrench committed Sep 5, 2024
2 parents d7c39cb + 2be1415 commit 76ca5d1
Show file tree
Hide file tree
Showing 209 changed files with 3,185 additions and 1,157 deletions.
5 changes: 4 additions & 1 deletion .github/ensure-tester/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,10 @@ runs:

- name: Print Startup Log
shell: bash
run: cat /run/log.out || true # don't fail
run:
# Try 6 times to see if .user-data-finished has been written to
for i in {0..6} ; do scripts/run_on_tester "cat /home/ubuntu/.user-data-finished" && break ; sleep 5 ; done ;
scripts/run_on_tester "cat /run/log.out"

# Set up a context for this run
- name: Copy Repo to Tester
Expand Down
13 changes: 10 additions & 3 deletions .github/spot-runner-action/dist/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,8 @@ class Ec2Instance {
Ebs: {
VolumeSize: 64,
VolumeType: 'gp3',
Throughput: 1000,
Iops: 5000
Throughput: 250,
Iops: 3000
},
},
],
Expand Down Expand Up @@ -1040,7 +1040,14 @@ class UserData {
`sudo service docker restart`,
"sudo wget -q https://github.com/earthly/earthly/releases/download/v0.8.10/earthly-linux-$(dpkg --print-architecture) -O /usr/local/bin/earthly",
"sudo chmod +x /usr/local/bin/earthly",
`sudo bash -c 'echo \'Acquire::Retries "3"; Acquire::https::Timeout "240"; Acquire::http::Timeout "240"; APT::Get::Assume-Yes "true"; APT::Install-Recommends "false"; APT::Install-Suggests "false";\' > /etc/apt/apt.conf.d/99-aztec-build'`,
`sudo bash -c 'cat <<EOF > /etc/apt/apt.conf.d/99-aztec-build
Acquire::Retries "3";
Acquire::https::Timeout "240";
Acquire::http::Timeout "240";
APT::Get::Assume-Yes "true";
APT::Install-Recommends "false";
APT::Install-Suggests "false";
EOF'`,
"sudo apt install -y brotli",
'echo "MaxStartups 1000" >> /etc/ssh/sshd_config',
'echo "ClientAliveInterval=30" >> /etc/ssh/sshd_config',
Expand Down
4 changes: 2 additions & 2 deletions .github/spot-runner-action/src/ec2.ts
Original file line number Diff line number Diff line change
Expand Up @@ -200,8 +200,8 @@ export class Ec2Instance {
Ebs: {
VolumeSize: 64,
VolumeType: 'gp3',
Throughput: 1000,
Iops: 5000
Throughput: 250,
Iops: 3000
},
},
],
Expand Down
9 changes: 8 additions & 1 deletion .github/spot-runner-action/src/userdata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,14 @@ export class UserData {
`sudo service docker restart`,
"sudo wget -q https://github.com/earthly/earthly/releases/download/v0.8.10/earthly-linux-$(dpkg --print-architecture) -O /usr/local/bin/earthly",
"sudo chmod +x /usr/local/bin/earthly",
`sudo bash -c 'echo \'Acquire::Retries "3"; Acquire::https::Timeout "240"; Acquire::http::Timeout "240"; APT::Get::Assume-Yes "true"; APT::Install-Recommends "false"; APT::Install-Suggests "false";\' > /etc/apt/apt.conf.d/99-aztec-build'`,
`sudo bash -c 'cat <<EOF > /etc/apt/apt.conf.d/99-aztec-build
Acquire::Retries "3";
Acquire::https::Timeout "240";
Acquire::http::Timeout "240";
APT::Get::Assume-Yes "true";
APT::Install-Recommends "false";
APT::Install-Suggests "false";
EOF'`,
"sudo apt install -y brotli",
'echo "MaxStartups 1000" >> /etc/ssh/sshd_config',
'echo "ClientAliveInterval=30" >> /etc/ssh/sshd_config',
Expand Down
31 changes: 0 additions & 31 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -161,37 +161,6 @@ jobs:
export FORCE_COLOR=1
../../scripts/earthly-ci -P --no-output +${{ matrix.test }}
network-e2e:
needs: [build, changes]
if: ${{ needs.changes.outputs.non-barretenberg-cpp == 'true' }}
runs-on: ubuntu-20.04
strategy:
matrix:
values_file: ["default.yaml", "3-validators.yaml"]
steps:
- uses: actions/checkout@v4
with: { ref: "${{ env.GIT_COMMIT }}" }
- uses: ./.github/ci-setup-action
- name: Setup and Test
uses: ./.github/ensure-tester-with-images
timeout-minutes: 45
with:
runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }}
builder_type: builder-x86
# these are copied to the tester and expected by the earthly command below
# if they fail to copy, it will try to build them on the tester and fail
builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
# command to produce the images in case they don't exist
builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
tester_ttl: 40
run: |
set -eux
cd ./yarn-project/end-to-end/
./scripts/setup_local_k8s.sh
export FORCE_COLOR=1
export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
../../scripts/earthly-ci --exec-stats -P --no-output ./+network-transfer --values-file=${{ matrix.values_file }}
# all the benchmarking end-to-end integration tests for aztec (not required to merge)
bench-e2e:
needs: [build, changes]
Expand Down
42 changes: 39 additions & 3 deletions .github/workflows/devnet-deploys.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ env:
TF_VAR_FORK_MNEMONIC: ${{ secrets.FORK_MNEMONIC }}
TF_VAR_INFURA_API_KEY: ${{ secrets.INFURA_API_KEY }}
TF_VAR_FORK_ADMIN_API_KEY: ${{ secrets.DEVNET_API_KEY }}
TF_VAR_MAINNET_FORK_CPU_UNITS: 2048
TF_VAR_MAINNET_FORK_MEMORY_UNITS: 4096

# Faucet
TF_VAR_FAUCET_ACCOUNT_INDEX: 9
Expand Down Expand Up @@ -123,6 +125,12 @@ jobs:
min_txs_per_block: ${{ steps.set_network_vars.outputs.min_txs_per_block }}
bot_flush_setup_txs: ${{ steps.set_network_vars.outputs.bot_flush_setup_txs }}
bot_max_pending_txs: ${{ steps.set_network_vars.outputs.bot_max_pending_txs }}
mainnet_fork_cpu_units: ${{ steps.set_network_vars.outputs.mainnet_fork_cpu_units }}
mainnet_fork_memory_units: ${{ steps.set_network_vars.outputs.mainnet_fork_memory_units }}
bot_skip_simulation: ${{ steps.set_network_vars.outputs.bot_skip_simulation }}
bot_l2_gas_limit: ${{ steps.set_network_vars.outputs.bot_l2_gas_limit }}
bot_da_gas_limit: ${{ steps.set_network_vars.outputs.bot_da_gas_limit }}
bot_count: ${{ steps.set_network_vars.outputs.bot_count }}
steps:
- name: Set network vars
shell: bash
Expand All @@ -135,7 +143,7 @@ jobs:
echo "branch_name=devnet" >> $GITHUB_OUTPUT
echo "network_api_key=DEVNET_API_KEY" >> $GITHUB_OUTPUT
echo "network_fork_admin_api_key=DEVNET_API_KEY" >> $GITHUB_OUTPUT
echo "agents_per_prover=4" >> $GITHUB_OUTPUT
echo "agents_per_prover=2" >> $GITHUB_OUTPUT
echo "bot_interval=180" >> $GITHUB_OUTPUT
echo "node_tcp_range_start=40100" >> $GITHUB_OUTPUT
echo "node_udp_range_start=45100" >> $GITHUB_OUTPUT
Expand All @@ -147,9 +155,15 @@ jobs:
echo "faucet_lb_priority=601" >> $GITHUB_OUTPUT
echo "min_txs_per_block=1" >> $GITHUB_OUTPUT
echo "max_txs_per_block=64" >> $GITHUB_OUTPUT
echo "bot_follow_chain=NONE" >> $GITHUB_OUTPUT
echo "bot_follow_chain=PROVEN" >> $GITHUB_OUTPUT
echo "bot_flush_setup_txs=false" >> $GITHUB_OUTPUT
echo "bot_max_pending_txs=1" >> $GITHUB_OUTPUT
echo "mainnet_fork_cpu_units=2048" >> $GITHUB_OUTPUT
echo "mainnet_fork_memory_units=4096" >> $GITHUB_OUTPUT
echo "bot_skip_simulation=false" >> $GITHUB_OUTPUT
echo "bot_l2_gas_limit=" >> $GITHUB_OUTPUT
echo "bot_da_gas_limit=" >> $GITHUB_OUTPUT
echo "bot_count=1" >> $GITHUB_OUTPUT
elif [ "$BRANCH_NAME" = "provernet" ]
then
echo "deploy_tag=provernet" >> $GITHUB_OUTPUT
Expand All @@ -171,14 +185,20 @@ jobs:
echo "bot_follow_chain=NONE" >> $GITHUB_OUTPUT
echo "bot_flush_setup_txs=true" >> $GITHUB_OUTPUT
echo "bot_max_pending_txs=32" >> $GITHUB_OUTPUT
echo "mainnet_fork_cpu_units=8192" >> $GITHUB_OUTPUT
echo "mainnet_fork_memory_units=32768" >> $GITHUB_OUTPUT
echo "bot_skip_simulation=true" >> $GITHUB_OUTPUT
echo "bot_l2_gas_limit=1000000000" >> $GITHUB_OUTPUT
echo "bot_da_gas_limit=1000000000" >> $GITHUB_OUTPUT
echo "bot_count=1" >> $GITHUB_OUTPUT
elif [ "$BRANCH_NAME" = "alphanet" ]
then
echo "deploy_tag=alphanet" >> $GITHUB_OUTPUT
echo "branch_name=alphanet" >> $GITHUB_OUTPUT
echo "network_api_key=ALPHANET_API_KEY" >> $GITHUB_OUTPUT
echo "network_fork_admin_api_key=ALPHANET_API_KEY" >> $GITHUB_OUTPUT
echo "agents_per_prover=1" >> $GITHUB_OUTPUT
echo "bot_interval=30" >> $GITHUB_OUTPUT
echo "bot_interval=10" >> $GITHUB_OUTPUT
echo "node_tcp_range_start=40000" >> $GITHUB_OUTPUT
echo "node_udp_range_start=45000" >> $GITHUB_OUTPUT
echo "prover_node_tcp_range_start=41000" >> $GITHUB_OUTPUT
Expand All @@ -192,6 +212,12 @@ jobs:
echo "bot_follow_chain=PROVEN" >> $GITHUB_OUTPUT
echo "bot_flush_setup_txs=false" >> $GITHUB_OUTPUT
echo "bot_max_pending_txs=1" >> $GITHUB_OUTPUT
echo "mainnet_fork_cpu_units=2048" >> $GITHUB_OUTPUT
echo "mainnet_fork_memory_units=4096" >> $GITHUB_OUTPUT
echo "bot_skip_simulation=false" >> $GITHUB_OUTPUT
echo "bot_l2_gas_limit=" >> $GITHUB_OUTPUT
echo "bot_da_gas_limit=" >> $GITHUB_OUTPUT
echo "bot_count=1" >> $GITHUB_OUTPUT
else
echo "Unrecognized Branch!!"
exit 1
Expand Down Expand Up @@ -462,6 +488,12 @@ jobs:
TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.prover_node_lb_priority_range_start }}
TF_VAR_SEQ_MIN_TX_PER_BLOCK: 1
TF_VAR_SEQ_MAX_TX_PER_BLOCK: ${{ needs.set-network.outputs.max_txs_per_block }}
TF_VAR_MAINNET_FORK_CPU_UNITS: ${{ needs.set-network.outputs.mainnet_fork_cpu_units }}
TF_VAR_MAINNET_FORK_MEMORY_UNITS: ${{ needs.set-network.outputs.mainnet_fork_memory_units }}
TF_VAR_BOT_SKIP_PUBLIC_SIMULATION: ${{ needs.set-network.outputs.bot_skip_simulation }}
TF_VAR_BOT_L2_GAS_LIMIT: ${{ needs.set-network.outputs.bot_l2_gas_limit }}
TF_VAR_BOT_DA_GAS_LIMIT: ${{ needs.set-network.outputs.bot_da_gas_limit }}
TF_VAR_BOT_COUNT: ${{ needs.set-network.outputs.bot_count }}
steps:
- uses: actions/checkout@v4
with:
Expand Down Expand Up @@ -679,6 +711,10 @@ jobs:
TF_VAR_BOT_FOLLOW_CHAIN: ${{ needs.set-network.outputs.bot_follow_chain }}
TF_VAR_PROVING_ENABLED: true
TF_VAR_BOT_NO_START: false
TF_VAR_BOT_SKIP_PUBLIC_SIMULATION: ${{ needs.set-network.outputs.bot_skip_simulation }}
TF_VAR_BOT_L2_GAS_LIMIT: ${{ needs.set-network.outputs.bot_l2_gas_limit }}
TF_VAR_BOT_DA_GAS_LIMIT: ${{ needs.set-network.outputs.bot_da_gas_limit }}
TF_VAR_BOT_COUNT: ${{ needs.set-network.outputs.bot_count }}
steps:
- uses: actions/checkout@v4
with:
Expand Down
175 changes: 175 additions & 0 deletions .github/workflows/spartan-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
name: Run spartan network tests
on:
workflow_dispatch:
schedule:
- cron: "00 08 * * 1-5"

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
GIT_COMMIT: ${{ github.sha }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

jobs:
setup:
uses: ./.github/workflows/setup-runner.yml
with:
username: ${{ github.event.pull_request.user.login || github.actor }}
runner_type: builder-x86
secrets: inherit

changes:
runs-on: ubuntu-20.04
# Required permissions.
permissions:
pull-requests: read
# Set job outputs to values from filter step
outputs:
avm-transpiler: ${{ steps.filter.outputs.avm-transpiler }}
build-images: ${{ steps.filter.outputs.build-images }}
barretenberg: ${{ steps.filter.outputs.barretenberg }}
barretenberg-cpp: ${{ steps.filter.outputs.barretenberg-cpp }}
noir: ${{ steps.filter.outputs.noir }}
noir-projects: ${{ steps.filter.outputs.noir-projects }}
l1-contracts: ${{ steps.filter.outputs.l1-contracts }}
non-docs: ${{ steps.filter.outputs.non-docs }}
non-misc-ci: ${{ steps.filter.outputs.non-misc-ci }}
non-barretenberg-cpp: ${{ steps.filter.outputs.non-barretenberg-cpp }}
steps:
- uses: actions/checkout@v4
with: { ref: "${{ env.GIT_COMMIT }}" }
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
id: filter
with:
filters: |
barretenberg:
- 'barretenberg/**'
barretenberg-cpp:
- 'barretenberg/cpp/**'
build-images:
- 'build-images/**'
noir:
- 'noir/**'
avm-transpiler:
- 'avm-transpiler/**'
l1-contracts:
- 'l1-contracts/**'
noir-projects:
- 'noir-projects/**'
non-barretenberg-cpp:
- '!(barretenberg/cpp/**)'
# don't consider AVM stuff 'core bb'
- barretenberg/cpp/pil/**
- barretenberg/cpp/src/barretenberg/vm/**
- barretenberg/cpp/src/barretenberg/**/generated/*
- barretenberg/cpp/src/barretenberg/client_ivc.{hpp,cpp}
non-docs:
- '!(docs/**)'
non-misc-ci:
- '!(.github/**)'
# Always rebuild when this file changes
- .github/workflows/ci.yml
build-images:
needs: [setup, changes]
# Note we don't but the 'if' here as that would also apply to dependent jobs, instead we just copy it into each step
runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
steps:
- uses: actions/checkout@v4
if: ${{ needs.changes.outputs.build-images == 'true' }}
with: { ref: "${{ env.GIT_COMMIT }}" }
- uses: ./.github/ci-setup-action
if: ${{ needs.changes.outputs.build-images == 'true' }}
with:
concurrency_key: build-images-x86
- name: "Push Build Images If Changed"
if: ${{ needs.changes.outputs.build-images == 'true' }}
timeout-minutes: 40
run: |
earthly-ci --push ./build-images/+build
build:
needs: [build-images, changes]
if: ${{ needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true' }}
runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
outputs:
e2e_list: ${{ steps.e2e_list.outputs.list }}
bench_list: ${{ steps.bench_list.outputs.list }}
steps:
- uses: actions/checkout@v4
with: { ref: "${{ env.GIT_COMMIT }}" }
- uses: ./.github/ci-setup-action
with:
concurrency_key: build-x86
# prepare images locally, tagged by commit hash
- name: "Build E2E Image"
timeout-minutes: 40
run: |
earthly-ci ./yarn-project+export-e2e-test-images
network-e2e:
needs: [build, changes]
if: ${{ needs.changes.outputs.non-barretenberg-cpp == 'true' }}
runs-on: ubuntu-20.04
strategy:
matrix:
values_file: ["default.yaml", "3-validators.yaml"]
steps:
- uses: actions/checkout@v4
with: { ref: "${{ env.GIT_COMMIT }}" }
- uses: ./.github/ci-setup-action
- name: Setup and Test
uses: ./.github/ensure-tester-with-images
timeout-minutes: 45
with:
runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }}
builder_type: builder-x86
# these are copied to the tester and expected by the earthly command below
# if they fail to copy, it will try to build them on the tester and fail
builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
# command to produce the images in case they don't exist
builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
tester_ttl: 40
run: |
set -eux
cd ./yarn-project/end-to-end/
./scripts/setup_local_k8s.sh
export FORCE_COLOR=1
export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
../../scripts/earthly-ci --exec-stats -P --no-output ./+network-transfer --values-file=${{ matrix.values_file }}
success-check:
runs-on: ubuntu-20.04
needs:
- build-and-test
if: always()
steps:
- name: Report overall success
env:
# We treat any skipped or failing jobs as a failure for the workflow as a whole.
FAIL: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
run: |
if [[ $FAIL == true ]]; then
echo "Test failed."
exit 1
fi
notify:
needs:
- success-check
runs-on: ubuntu-20.04
if: ${{ github.ref == 'refs/heads/master' && failure() }}
steps:
- name: Send notification to aztec3-ci channel if workflow failed on master
uses: slackapi/[email protected]
with:
payload: |
{
"url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WORKFLOW_TRIGGER_URL }}
Loading

0 comments on commit 76ca5d1

Please sign in to comment.