From 906ef4d62b758e5c8573afd43b089d8deffd92b9 Mon Sep 17 00:00:00 2001
From: Mitch <mitchell@aztecprotocol.com>
Date: Thu, 31 Oct 2024 11:40:54 -0400
Subject: [PATCH 1/6] feat: add a test for reorgs on k8s

---
 .github/workflows/ci.yml                      |   4 +-
 .github/workflows/nightly-kind-test.yml       |   8 +-
 spartan/.rebuild_patterns                     |   1 +
 spartan/Earthfile                             |   8 +
 .../aztec-network/templates/prover-node.yaml  |  52 +++-
 spartan/aztec-network/templates/pxe.yaml      |   6 +-
 .../templates/transaction-bot.yaml            |   4 +
 spartan/aztec-network/values.yaml             |  15 +-
 .../values/16-validators-with-metrics.yaml    |  55 +++++
 .../aztec-network/values/16-validators.yaml   |  16 --
 .../values/3-validators-with-metrics.yaml     |   2 -
 .../aztec-network/values/3-validators.yaml    |   2 -
 spartan/chaos-mesh/install.sh                 |  12 +-
 spartan/metrics/install.sh                    |  11 +-
 .../scripts/apply_network_shaping.sh          |   3 -
 .../templates/network-chaos.yaml              |  29 ++-
 spartan/network-shaping/values.yaml           |   8 +-
 .../network-shaping/values/kill-provers.yaml  |  17 ++
 spartan/scripts/setup_local_k8s.sh            |   5 +
 yarn-project/Earthfile                        |  15 +-
 yarn-project/end-to-end/package.json          |   3 +-
 .../end-to-end/scripts/e2e_test_config.yml    |   2 +-
 .../scripts/native-network/test-transfer.sh   |   1 +
 .../end-to-end/scripts/native_network_test.sh |   2 +
 .../end-to-end/scripts/network_test.sh        |  23 +-
 .../end-to-end/src/spartan/4epochs.test.ts    |  31 ++-
 .../end-to-end/src/spartan/k8_utils.ts        | 222 ++++++++++++++++++
 .../end-to-end/src/spartan/reorg.test.ts      | 123 ++++++++++
 .../src/spartan/setup_test_wallets.ts         |  30 +++
 .../end-to-end/src/spartan/smoke.test.ts      |  20 +-
 .../end-to-end/src/spartan/transfer.test.ts   |  18 +-
 yarn-project/yarn.lock                        |   3 +-
 32 files changed, 671 insertions(+), 80 deletions(-)
 create mode 100644 spartan/.rebuild_patterns
 create mode 100644 spartan/Earthfile
 create mode 100644 spartan/aztec-network/values/16-validators-with-metrics.yaml
 delete mode 100644 spartan/network-shaping/scripts/apply_network_shaping.sh
 create mode 100644 spartan/network-shaping/values/kill-provers.yaml
 create mode 100644 yarn-project/end-to-end/src/spartan/k8_utils.ts
 create mode 100644 yarn-project/end-to-end/src/spartan/reorg.test.ts

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 13c81faeb90..0563cfe90e8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -664,7 +664,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: [smoke.test.ts, transfer.test.ts] # TODO reinstate: 4epochs.test.ts
+        test: [smoke.test.ts, transfer.test.ts, reorg.test.ts]
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -691,7 +691,7 @@ jobs:
             cd yarn-project/end-to-end
             echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
             test=${{ matrix.test }}
-            NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=${values_file:-default.yaml} ./scripts/network_test.sh ./src/spartan/$test
+            NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=16-validators.yaml ./scripts/network_test.sh ./src/spartan/$test
 
   l1-contracts-test:
     needs: [build, configure]
diff --git a/.github/workflows/nightly-kind-test.yml b/.github/workflows/nightly-kind-test.yml
index 668dc341ca6..3ed6b5f6367 100644
--- a/.github/workflows/nightly-kind-test.yml
+++ b/.github/workflows/nightly-kind-test.yml
@@ -53,14 +53,14 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        values_file: ["default.yaml", "3-validators.yaml"]
+        test: ["transfer", "reorg"]
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
       - name: Setup and Test
         uses: ./.github/ensure-tester-with-images
-        timeout-minutes: 45
+        timeout-minutes: 90
         with:
           runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }}
           builder_type: builder-x86
@@ -69,12 +69,12 @@ jobs:
           builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
           # command to produce the images in case they don't exist
           builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
-          tester_ttl: 40
+          tester_ttl: 90
           run: |
             set -eux
             ./spartan/scripts/setup_local_k8s.sh
             export FORCE_COLOR=1
-            NAMESPACE=transfer FRESH_INSTALL=true VALUES_FILE=${{ matrix.values_file }} ./scripts/network_test.sh ./src/spartan/transfer.test.ts || true
+            NAMESPACE=${{ matrix.test }} FRESH_INSTALL=true VALUES_FILE="16-validators.yaml" ./scripts/network_test.sh ./src/spartan/${{ matrix.test }}.test.ts || true
 
   success-check:
     runs-on: ubuntu-20.04
diff --git a/spartan/.rebuild_patterns b/spartan/.rebuild_patterns
new file mode 100644
index 00000000000..14a26919954
--- /dev/null
+++ b/spartan/.rebuild_patterns
@@ -0,0 +1 @@
+^spartan/.*$
diff --git a/spartan/Earthfile b/spartan/Earthfile
new file mode 100644
index 00000000000..04ac1aadc4f
--- /dev/null
+++ b/spartan/Earthfile
@@ -0,0 +1,8 @@
+VERSION 0.8
+
+charts:
+    ARG EARTHLY_GIT_HASH
+    FROM ubuntu:noble
+    WORKDIR /usr/src
+    COPY . ./spartan
+    SAVE ARTIFACT /usr/src /usr/src
diff --git a/spartan/aztec-network/templates/prover-node.yaml b/spartan/aztec-network/templates/prover-node.yaml
index 95067f0a61e..486a29c33de 100644
--- a/spartan/aztec-network/templates/prover-node.yaml
+++ b/spartan/aztec-network/templates/prover-node.yaml
@@ -93,12 +93,31 @@ spec:
             - name: PROVER_PUBLISHER_PRIVATE_KEY
               value: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
             # get private proofs from the boot node
-            - name: PROVER_COORDINATION_NODE_URL
-              value: {{ include "aztec-network.bootNodeUrl" . | quote }}
             - name: PROVER_JOB_SOURCE_URL
               value: "http://$(POD_IP):{{ .Values.proverNode.service.nodePort }}"
+            - name: P2P_ENABLED
+              value: "{{ .Values.proverNode.p2pEnabled }}"
+            - name: P2P_TCP_ANNOUNCE_ADDR
+              {{- if .Values.proverNode.externalTcpHost }}
+              value: "{{ .Values.proverNode.externalTcpHost }}:{{ .Values.proverNode.service.p2pTcpPort }}"
+              {{- else }}
+              value: "$(POD_DNS_NAME):{{ .Values.proverNode.service.p2pTcpPort }}"
+              {{- end }}
+            - name: P2P_UDP_ANNOUNCE_ADDR
+              {{- if .Values.proverNode.externalUdpHost }}
+              value: "{{ .Values.proverNode.externalUdpHost }}:{{ .Values.proverNode.service.p2pUdpPort }}"
+              {{- else }}
+              value: "$(POD_DNS_NAME):{{ .Values.proverNode.service.p2pUdpPort }}"
+              {{- end }}
+            - name: P2P_TCP_LISTEN_ADDR
+              value: "0.0.0.0:{{ .Values.proverNode.service.p2pTcpPort }}"
+            - name: P2P_UDP_LISTEN_ADDR
+              value: "0.0.0.0:{{ .Values.proverNode.service.p2pUdpPort }}"
           ports:
             - containerPort: {{ .Values.proverNode.service.nodePort }}
+            - containerPort: {{ .Values.proverNode.service.p2pTcpPort }}
+            - containerPort: {{ .Values.proverNode.service.p2pUdpPort }}
+              protocol: UDP
           resources:
             {{- toYaml .Values.proverNode.resources | nindent 12 }}
       volumes:
@@ -140,12 +159,17 @@ spec:
   ports:
     - port: {{ .Values.proverNode.service.nodePort }}
       name: node
+    - port: {{ .Values.proverNode.service.p2pTcpPort }}
+      name: p2p-tcp
+    - port: {{ .Values.proverNode.service.p2pUdpPort }}
+      name: p2p-udp
+      protocol: UDP
 ---
 {{if .Values.network.public }}
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "aztec-network.fullname" . }}-prover-node-lb
+  name: {{ include "aztec-network.fullname" . }}-prover-node-lb-tcp
   labels:
     {{- include "aztec-network.labels" . | nindent 4 }}
 spec:
@@ -156,5 +180,27 @@ spec:
   ports:
     - port: {{ .Values.proverNode.service.nodePort }}
       name: node
+    - port: {{ .Values.proverNode.service.p2pTcpPort }}
+      name: p2p-tcp
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "aztec-network.fullname" . }}-prover-node-lb-udp
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
+    service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"
+  labels:
+    {{- include "aztec-network.labels" . | nindent 4 }}
+spec:
+  type: LoadBalancer
+  selector:
+    {{- include "aztec-network.selectorLabels" . | nindent 4 }}
+    app: prover-node
+  ports:
+    - port: {{ .Values.proverNode.service.p2pUdpPort }}
+      name: p2p-udp
+      protocol: UDP
 {{ end }}
 {{ end }}
diff --git a/spartan/aztec-network/templates/pxe.yaml b/spartan/aztec-network/templates/pxe.yaml
index d885313a6f8..c30dee8b093 100644
--- a/spartan/aztec-network/templates/pxe.yaml
+++ b/spartan/aztec-network/templates/pxe.yaml
@@ -70,10 +70,8 @@ spec:
                 - -c
                 - |
                   curl -s -X POST -H 'content-type: application/json' \
-                  -d '{"jsonrpc":"2.0","method":"pxe_getNodeInfo","params":[],"id":67}' \
-                  127.0.0.1:{{ .Values.pxe.service.port }} > /tmp/probe_output.txt && \
-                  cat /tmp/probe_output.txt && \
-                  grep -q '"enr:-' /tmp/probe_output.txt
+                  -d '{"jsonrpc":"2.0","method":"pxe_isGlobalStateSynchronized","params":[],"id":67}' \
+                  127.0.0.1:{{ .Values.pxe.service.port }} | grep -q '"result":true'
             initialDelaySeconds: {{ .Values.pxe.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.pxe.readinessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.pxe.readinessProbe.timeoutSeconds }}
diff --git a/spartan/aztec-network/templates/transaction-bot.yaml b/spartan/aztec-network/templates/transaction-bot.yaml
index 655bd17fec5..75a1d09f32c 100644
--- a/spartan/aztec-network/templates/transaction-bot.yaml
+++ b/spartan/aztec-network/templates/transaction-bot.yaml
@@ -60,6 +60,10 @@ spec:
               value: "{{ .Values.bot.pxeProverEnabled }}"
             - name: PROVER_REAL_PROOFS
               value: "{{ .Values.bot.proverRealProofs }}"
+            - name: BOT_MAX_CONSECUTIVE_ERRORS
+              value: "3"
+            - name: BOT_STOP_WHEN_UNHEALTHY
+              value: "true"
           ports:
             - name: http
               containerPort: {{ .Values.bot.service.port }}
diff --git a/spartan/aztec-network/values.yaml b/spartan/aztec-network/values.yaml
index d8e29eb0a3f..8480dc61124 100644
--- a/spartan/aztec-network/values.yaml
+++ b/spartan/aztec-network/values.yaml
@@ -93,9 +93,13 @@ validator:
 
 proverNode:
   external: false
-  externalHost: ""
+  externalTcpHost: ""
+  externalUdpHost: ""
   replicas: 1
+  p2pEnabled: true
   service:
+    p2pTcpPort: 40400
+    p2pUdpPort: 40400
     nodePort: 8080
   logLevel: "debug"
   debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:world-state:database,-aztec:l2_block_stream*"
@@ -111,8 +115,7 @@ pxe:
   external: false
   externalHost: ""
   logLevel: "debug"
-  debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*"
-  proverEnable: false
+  debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:l2_block_stream,-aztec:world-state:database"
   replicas: 1
   service:
     port: 8080
@@ -132,12 +135,12 @@ pxe:
 bot:
   enabled: true
   logLevel: "debug"
-  debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*"
+  debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:l2_block_stream,-aztec:world-state:database"
   replicas: 1
   botPrivateKey: "0xcafe"
   txIntervalSeconds: 24
-  privateTransfersPerTx: 1
-  publicTransfersPerTx: 0
+  privateTransfersPerTx: 0
+  publicTransfersPerTx: 1
   # Do not wait for transactions
   followChain: "NONE"
   botNoStart: false
diff --git a/spartan/aztec-network/values/16-validators-with-metrics.yaml b/spartan/aztec-network/values/16-validators-with-metrics.yaml
new file mode 100644
index 00000000000..8bc8f2c115c
--- /dev/null
+++ b/spartan/aztec-network/values/16-validators-with-metrics.yaml
@@ -0,0 +1,55 @@
+##########
+# BEWARE #
+##########
+# You need to deploy the metrics helm chart before using this values file.
+# head to spartan/metrics and run `./install.sh`
+# (then `./forward.sh` if you want to see it)
+telemetry:
+  enabled: true
+  otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318
+
+validator:
+  replicas: 16
+  validatorKeys:
+    - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
+    - 0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d
+    - 0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a
+    - 0x7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6
+    - 0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a
+    - 0x8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba
+    - 0x92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e
+    - 0x4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356
+    - 0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97
+    - 0x2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6
+    - 0xf214f2b2cd398c806f84e317254e0f0b801d0643303237d97a22a48e01628897
+    - 0x701b615bbdfb9de65240bc28bd21bbc0d996645a3dd57e7b12bc2bdf6f192c82
+    - 0xa267530f49f8280200edf313ee7af6b827f2a8bce2897751d06a843f644967b1
+    - 0x47c99abed3324a2707c28affff1267e45918ec8c3f20b8aa892e8b065d2942dd
+    - 0xc526ee95bf44d8fc405a158bb884d9d1238d99f0612e9f33d006bb0789009aaa
+    - 0x8166f546bab6da521a8369cab06c5d2b9e46670292d85c875ee9ec20e84ffb61
+  validatorAddresses:
+    - 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266
+    - 0x70997970C51812dc3A010C7d01b50e0d17dc79C8
+    - 0x3C44CdDdB6a900fa2b585dd299e03d12FA4293BC
+    - 0x90F79bf6EB2c4f870365E785982E1f101E93b906
+    - 0x15d34AAf54267DB7D7c367839AAf71A00a2C6A65
+    - 0x9965507D1a55bcC2695C58ba16FB37d819B0A4dc
+    - 0x976EA74026E726554dB657fA54763abd0C3a0aa9
+    - 0x14dC79964da2C08b23698B3D3cc7Ca32193d9955
+    - 0x23618e81E3f5cdF7f54C3d65f7FBc0aBf5B21E8f
+    - 0xa0Ee7A142d267C1f36714E4a8F75612F20a79720
+    - 0xBcd4042DE499D14e55001CcbB24a551F3b954096
+    - 0x71bE63f3384f5fb98995898A86B02Fb2426c5788
+    - 0xFABB0ac9d68B0B445fB7357272Ff202C5651694a
+    - 0x1CBd3b2770909D4e10f157cABC84C7264073C9Ec
+    - 0xdF3e18d64BC6A983f673Ab319CCaE4f1a57C7097
+    - 0xcd3B766CCDd6AE721141F452C550Ca635964ce71
+  resources:
+    requests:
+      memory: "512Mi"
+  validator:
+    disabled: false
+
+bootNode:
+  validator:
+    disabled: true
diff --git a/spartan/aztec-network/values/16-validators.yaml b/spartan/aztec-network/values/16-validators.yaml
index b9a2f5e13f0..354027baec9 100644
--- a/spartan/aztec-network/values/16-validators.yaml
+++ b/spartan/aztec-network/values/16-validators.yaml
@@ -1,15 +1,4 @@
-##########
-# BEWARE #
-##########
-# You need to deploy the metrics helm chart before using this values file.
-# head to spartan/metrics and run `./install.sh`
-# (then `./forward.sh` if you want to see it)
-telemetry:
-  enabled: true
-  otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318
-
 validator:
-  debug: "aztec:*,-aztec:avm_simulator:*,-aztec:libp2p_service"
   replicas: 16
   validatorKeys:
     - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
@@ -51,11 +40,6 @@ validator:
   validator:
     disabled: false
 
-bot:
-  txIntervalSeconds: 2
-  privateTransfersPerTx: 1
-  publicTransfersPerTx: 2
-
 bootNode:
   validator:
     disabled: true
diff --git a/spartan/aztec-network/values/3-validators-with-metrics.yaml b/spartan/aztec-network/values/3-validators-with-metrics.yaml
index 8252a8e7998..b20b34b5194 100644
--- a/spartan/aztec-network/values/3-validators-with-metrics.yaml
+++ b/spartan/aztec-network/values/3-validators-with-metrics.yaml
@@ -9,7 +9,6 @@ telemetry:
   otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318
 
 validator:
-  debug: "aztec:*,-aztec:avm_simulator:*,-aztec:world-state:database,discv5:*,-JsonProxy:*"
   replicas: 3
   validatorKeys:
     - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
@@ -23,6 +22,5 @@ validator:
     disabled: false
 
 bootNode:
-  debug: "aztec:*,-aztec:avm_simulator:*,-aztec:world-state:database,discv5:*,-JsonProxy:*"
   validator:
     disabled: true
diff --git a/spartan/aztec-network/values/3-validators.yaml b/spartan/aztec-network/values/3-validators.yaml
index 073c82bdf5e..44ad268572e 100644
--- a/spartan/aztec-network/values/3-validators.yaml
+++ b/spartan/aztec-network/values/3-validators.yaml
@@ -1,5 +1,4 @@
 validator:
-  debug: "aztec:*,-aztec:avm_simulator:*,-aztec:libp2p_service"
   replicas: 3
   validatorKeys:
     - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
@@ -12,7 +11,6 @@ validator:
   validator:
     disabled: false
 
-
 bootNode:
   validator:
     disabled: true
diff --git a/spartan/chaos-mesh/install.sh b/spartan/chaos-mesh/install.sh
index 53a9e4793c1..52146403cd5 100755
--- a/spartan/chaos-mesh/install.sh
+++ b/spartan/chaos-mesh/install.sh
@@ -1,7 +1,15 @@
 #!/bin/bash
 
-# Install chaos-mesh
+SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
+cd "$SCRIPT_DIR"
+
+# check if chaos-mesh is already installed
+if helm ls --namespace chaos-mesh | grep -q chaos; then
+  echo "chaos-mesh is already installed"
+  exit 0
+fi
+
 helm repo add chaos-mesh https://charts.chaos-mesh.org
 helm dependency update
+helm upgrade chaos "$SCRIPT_DIR" -n chaos-mesh --install --create-namespace --atomic
 
-helm upgrade chaos . -n chaos-mesh --install --create-namespace --atomic
\ No newline at end of file
diff --git a/spartan/metrics/install.sh b/spartan/metrics/install.sh
index 2712e900756..66bd2cbd919 100755
--- a/spartan/metrics/install.sh
+++ b/spartan/metrics/install.sh
@@ -1,10 +1,17 @@
 #!/bin/bash
 set -eu
 
-cd "$(dirname "${BASH_SOURCE[0]}")"
+SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
+cd "$SCRIPT_DIR"
+
+# check if metrics is already installed
+if helm ls --namespace metrics | grep -q metrics; then
+  echo "metrics is already installed"
+  exit 0
+fi
 
 helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
 helm repo add grafana https://grafana.github.io/helm-charts
 helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 helm dependency update
-helm upgrade metrics . -n metrics --install --create-namespace --atomic
+helm upgrade metrics "$SCRIPT_DIR" -n metrics --install --create-namespace --atomic
diff --git a/spartan/network-shaping/scripts/apply_network_shaping.sh b/spartan/network-shaping/scripts/apply_network_shaping.sh
deleted file mode 100644
index becb34cf163..00000000000
--- a/spartan/network-shaping/scripts/apply_network_shaping.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-helm upgrade chaos-mesh . -n chaos-mesh --install --atomic
\ No newline at end of file
diff --git a/spartan/network-shaping/templates/network-chaos.yaml b/spartan/network-shaping/templates/network-chaos.yaml
index f18ee2b8d71..21c954767cb 100644
--- a/spartan/network-shaping/templates/network-chaos.yaml
+++ b/spartan/network-shaping/templates/network-chaos.yaml
@@ -4,7 +4,7 @@
 apiVersion: chaos-mesh.org/v1alpha1
 kind: NetworkChaos
 metadata:
-  name: {{ .Values.global.namespace }}-latency
+  name: {{ .Values.global.targetNamespace }}-latency
   namespace: {{ .Values.global.chaosMeshNamespace }}
   labels:
     {{- include "network-shaping.labels" . | nindent 4 }}
@@ -28,7 +28,7 @@ spec:
 apiVersion: chaos-mesh.org/v1alpha1
 kind: NetworkChaos
 metadata:
-  name: {{ .Values.global.namespace }}-bandwidth
+  name: {{ .Values.global.targetNamespace }}-bandwidth
   namespace: {{ .Values.global.chaosMeshNamespace }}
   labels:
     {{- include "network-shaping.labels" . | nindent 4 }}
@@ -54,7 +54,7 @@ spec:
 apiVersion: chaos-mesh.org/v1alpha1
 kind: NetworkChaos
 metadata:
-  name: {{ .Values.global.namespace }}-packet-loss
+  name: {{ .Values.global.targetNamespace }}-packet-loss
   namespace: {{ .Values.global.chaosMeshNamespace }}
   labels:
     {{- include "network-shaping.labels" . | nindent 4 }}
@@ -71,4 +71,27 @@ spec:
     correlation: {{ .Values.networkShaping.conditions.packetLoss.correlation | quote }}
   duration: 8760h
 {{- end }}
+
+{{- if .Values.networkShaping.conditions.killProvers.enabled }}
+---
+apiVersion: chaos-mesh.org/v1alpha1
+kind: PodChaos
+metadata:
+  name: {{ .Values.global.targetNamespace }}-kill-provers
+  namespace: {{ .Values.global.chaosMeshNamespace }}
+  labels:
+    {{- include "network-shaping.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/resource-policy": keep
+spec:
+  action: pod-failure
+  mode: all
+  selector:
+    namespaces:
+      - {{ .Values.global.targetNamespace }}
+    labelSelectors:
+      app: prover-node
+  duration: {{ .Values.networkShaping.conditions.killProvers.duration }}
+{{- end }}
+
 {{- end }}
\ No newline at end of file
diff --git a/spartan/network-shaping/values.yaml b/spartan/network-shaping/values.yaml
index 4518c645fbe..0b3ed0d698e 100644
--- a/spartan/network-shaping/values.yaml
+++ b/spartan/network-shaping/values.yaml
@@ -35,7 +35,6 @@ networkShaping:
         # Eg: 75 means the current delay will be 75% influenced by the previous delay
         correlation: "75"
 
-
     packetLoss:
       # Enable / disable packet loss configuration
       enabled: false
@@ -63,8 +62,9 @@ networkShaping:
       # Buffer = smoother bandwidth restriction but higher memory usage
       buffer: 1000
 
-
-
+    killProvers:
+      enabled: false
+      duration: 13m
 ## Here are some exciting example configurations created by claude:
 # Example use cases for different configurations:
 
@@ -99,4 +99,4 @@ networkShaping:
 #   delay:
 #     latency: 150ms
 #     jitter: 30ms
-#     correlation: "75"
\ No newline at end of file
+#     correlation: "75"
diff --git a/spartan/network-shaping/values/kill-provers.yaml b/spartan/network-shaping/values/kill-provers.yaml
new file mode 100644
index 00000000000..2128efca00a
--- /dev/null
+++ b/spartan/network-shaping/values/kill-provers.yaml
@@ -0,0 +1,17 @@
+# Simulates congested network conditions
+# High latency, limited bandwidth, packet loss
+global:
+  namespace: "smoke"
+
+networkShaping:
+  enabled: true
+  conditions:
+    latency:
+      enabled: false
+    bandwidth:
+      enabled: false
+    packetLoss:
+      enabled: false
+    killProvers:
+      enabled: true
+      duration: 13m
diff --git a/spartan/scripts/setup_local_k8s.sh b/spartan/scripts/setup_local_k8s.sh
index 089aba708a7..431600dc558 100755
--- a/spartan/scripts/setup_local_k8s.sh
+++ b/spartan/scripts/setup_local_k8s.sh
@@ -2,6 +2,8 @@
 
 set -e
 
+SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
+
 # exit if we are not on linux amd64
 if [ "$(uname)" != "Linux" ] || [ "$(uname -m)" != "x86_64" ]; then
   echo "This script is only supported on Linux amd64"
@@ -58,3 +60,6 @@ else
 fi
 
 kubectl config use-context kind-kind || true
+
+"$SCRIPT_DIR"/../chaos-mesh/install.sh
+"$SCRIPT_DIR"/../metrics/install.sh
diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile
index fb3cb799b63..98ab740d010 100644
--- a/yarn-project/Earthfile
+++ b/yarn-project/Earthfile
@@ -190,10 +190,13 @@ export-aztec-faucet:
 
 # We care about creating a slimmed down e2e image because we have to serialize it from earthly to docker for running.
 end-to-end-prod:
+    BUILD ../spartan/+charts
     FROM +cli-base
     RUN yarn workspaces focus @aztec/end-to-end @aztec/cli-wallet --production && yarn cache clean
     COPY --dir +rollup-verifier-contract/usr/src/bb /usr/src
     COPY --dir +build-dev/usr/src/noir-projects/noir-contracts /usr/src/noir-projects/noir-contracts
+    COPY --dir ../spartan/+charts/usr/src/spartan /usr/src/spartan
+
     SAVE ARTIFACT /usr/src /usr/src
 
 anvil:
@@ -209,9 +212,17 @@ end-to-end-base:
         && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
         && echo "deb [arch=$(dpkg --print-architecture)] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \
         && apt update && apt install curl chromium nodejs netcat-openbsd -y \
-        && rm -rf /var/lib/apt/lists/*
+        && rm -rf /var/lib/apt/lists/* \
+        && mkdir -p /usr/local/bin \
+        && curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
+        && chmod +x /usr/local/bin/kubectl \
+        && curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 \
+        && chmod +x get_helm.sh \
+        && ./get_helm.sh \
+        && rm get_helm.sh
+
     ENV CHROME_BIN="/usr/bin/chromium"
-    ENV PATH=/opt/foundry/bin:$PATH
+    ENV PATH=/opt/foundry/bin:/usr/local/bin:$PATH
     ENV HARDWARE_CONCURRENCY=""
     ENV FAKE_PROOFS=""
     ENV BB_WORKING_DIRECTORY=/usr/src/bb
diff --git a/yarn-project/end-to-end/package.json b/yarn-project/end-to-end/package.json
index d2819649d11..76c4307dbf6 100644
--- a/yarn-project/end-to-end/package.json
+++ b/yarn-project/end-to-end/package.json
@@ -92,7 +92,8 @@
     "viem": "^2.7.15",
     "webpack": "^5.88.2",
     "webpack-cli": "^5.1.4",
-    "winston": "^3.10.0"
+    "winston": "^3.10.0",
+    "zod": "^3.23.8"
   },
   "devDependencies": {
     "0x": "^5.7.0",
diff --git a/yarn-project/end-to-end/scripts/e2e_test_config.yml b/yarn-project/end-to-end/scripts/e2e_test_config.yml
index 55939e7e8ac..70e719e1724 100644
--- a/yarn-project/end-to-end/scripts/e2e_test_config.yml
+++ b/yarn-project/end-to-end/scripts/e2e_test_config.yml
@@ -119,7 +119,7 @@ tests:
       NAMESPACE: 'transfer'
       FRESH_INSTALL: 'true'
       VALUES_FILE: '$-default.yaml'
-    command: './scripts/network_test.sh ./src/spartan/smoke.test.ts'
+    command: './scripts/network_test.sh ./src/spartan/transfer.test.ts'
   pxe:
     use_compose: true
   uniswap_trade_on_l1_from_l2:
diff --git a/yarn-project/end-to-end/scripts/native-network/test-transfer.sh b/yarn-project/end-to-end/scripts/native-network/test-transfer.sh
index fe9a8fc2004..50790afbe3e 100755
--- a/yarn-project/end-to-end/scripts/native-network/test-transfer.sh
+++ b/yarn-project/end-to-end/scripts/native-network/test-transfer.sh
@@ -10,6 +10,7 @@ exec > >(tee -a "$(dirname $0)/logs/${SCRIPT_NAME}.log") 2> >(tee -a "$(dirname
 
 export BOOTNODE_URL=${BOOTNODE_URL:-http://127.0.0.1:8080}
 export PXE_URL=${PXE_URL:-http://127.0.0.1:8079}
+export ETHEREUM_HOST=${ETHEREUM_HOST:-http://127.0.0.1:8545}
 
 REPO=$(git rev-parse --show-toplevel)
 # Run our test assuming the port in pxe.sh
diff --git a/yarn-project/end-to-end/scripts/native_network_test.sh b/yarn-project/end-to-end/scripts/native_network_test.sh
index 3c95833b4a5..846490be0f1 100755
--- a/yarn-project/end-to-end/scripts/native_network_test.sh
+++ b/yarn-project/end-to-end/scripts/native_network_test.sh
@@ -45,6 +45,8 @@ function run_parallel() {
   fi
 }
 
+export K8S=false
+
 # We exit with the return code of the first command
 # While the others are ran in the background, either in tmux or just interleaved
 run_parallel "$@"
\ No newline at end of file
diff --git a/yarn-project/end-to-end/scripts/network_test.sh b/yarn-project/end-to-end/scripts/network_test.sh
index c5969fbf1b3..83b3dc463ed 100755
--- a/yarn-project/end-to-end/scripts/network_test.sh
+++ b/yarn-project/end-to-end/scripts/network_test.sh
@@ -106,13 +106,19 @@ function cleanup() {
   trap - SIGTERM && kill $(pgrep -g $$ | grep -v $$) $(jobs -p) &>/dev/null || true
 }
 trap cleanup SIGINT SIGTERM EXIT
+
+# if we don't have a chaos values, remove any existing chaos experiments
+if [ -z "${CHAOS_VALUES:-}" ]; then
+  echo "Deleting existing network chaos experiments..."
+  kubectl delete networkchaos --all --all-namespaces
+fi
+
 # Install the Helm chart
 helm upgrade --install spartan "$REPO/spartan/aztec-network/" \
       --namespace "$NAMESPACE" \
       --create-namespace \
       --values "$REPO/spartan/aztec-network/values/$VALUES_FILE" \
       --set images.aztec.image="aztecprotocol/aztec:$AZTEC_DOCKER_TAG" \
-      --set ingress.enabled=true \
       --wait \
       --wait-for-jobs=true \
       --timeout=30m
@@ -140,14 +146,17 @@ if ! handle_network_shaping; then
   fi
 fi
 
-# Start port-forwarding with dynamically allocated free ports
-(kubectl port-forward --namespace $NAMESPACE svc/spartan-aztec-network-pxe $PXE_PORT:8080 2>/dev/null >/dev/null || true) &
-(kubectl port-forward --namespace $NAMESPACE svc/spartan-aztec-network-ethereum $ANVIL_PORT:8545 2>/dev/null >/dev/null || true) &
 
 docker run --rm --network=host \
-  -e PXE_URL=http://127.0.0.1:$PXE_PORT \
+  -v ~/.kube:/root/.kube \
+  -e K8S=true \
+  -e SPARTAN_DIR="/usr/src/spartan" \
+  -e NAMESPACE="$NAMESPACE" \
+  -e HOST_PXE_PORT=$PXE_PORT \
+  -e CONTAINER_PXE_PORT=8080 \
+  -e HOST_ETHEREUM_PORT=$ANVIL_PORT \
+  -e CONTAINER_ETHEREUM_PORT=8545 \
   -e DEBUG="aztec:*" \
-  -e LOG_LEVEL=debug \
-  -e ETHEREUM_HOST=http://127.0.0.1:$ANVIL_PORT \
   -e LOG_JSON=1 \
+  -e LOG_LEVEL=debug \
   aztecprotocol/end-to-end:$AZTEC_DOCKER_TAG $TEST
diff --git a/yarn-project/end-to-end/src/spartan/4epochs.test.ts b/yarn-project/end-to-end/src/spartan/4epochs.test.ts
index 8ddfce815fc..3ddd2304d03 100644
--- a/yarn-project/end-to-end/src/spartan/4epochs.test.ts
+++ b/yarn-project/end-to-end/src/spartan/4epochs.test.ts
@@ -6,15 +6,10 @@ import { TokenContract } from '@aztec/noir-contracts.js';
 import { jest } from '@jest/globals';
 
 import { RollupCheatCodes } from '../../../aztec.js/src/utils/cheat_codes.js';
+import { getConfig, isK8sConfig, startPortForward } from './k8_utils.js';
 import { type TestWallets, setupTestWalletsWithTokens } from './setup_test_wallets.js';
 
-const { PXE_URL, ETHEREUM_HOST } = process.env;
-if (!PXE_URL) {
-  throw new Error('PXE_URL env variable must be set');
-}
-if (!ETHEREUM_HOST) {
-  throw new Error('ETHEREUM_HOST env variable must be set');
-}
+const config = getConfig(process.env);
 
 describe('token transfer test', () => {
   jest.setTimeout(10 * 60 * 4000); // 40 minutes
@@ -26,8 +21,30 @@ describe('token transfer test', () => {
   const ROUNDS = BigInt(AZTEC_EPOCH_DURATION * TEST_EPOCHS);
 
   let testWallets: TestWallets;
+  let PXE_URL: string;
+  let ETHEREUM_HOST: string;
 
   beforeAll(async () => {
+    if (isK8sConfig(config)) {
+      await startPortForward({
+        resource: 'svc/spartan-aztec-network-pxe',
+        namespace: config.NAMESPACE,
+        containerPort: config.CONTAINER_PXE_PORT,
+        hostPort: config.HOST_PXE_PORT,
+      });
+      await startPortForward({
+        resource: 'svc/spartan-aztec-network-ethereum',
+        namespace: config.NAMESPACE,
+        containerPort: config.CONTAINER_ETHEREUM_PORT,
+        hostPort: config.HOST_ETHEREUM_PORT,
+      });
+      PXE_URL = `http://127.0.0.1:${config.HOST_PXE_PORT}`;
+      ETHEREUM_HOST = `http://127.0.0.1:${config.HOST_ETHEREUM_PORT}`;
+    } else {
+      PXE_URL = config.PXE_URL;
+      ETHEREUM_HOST = config.ETHEREUM_HOST;
+    }
+
     testWallets = await setupTestWalletsWithTokens(PXE_URL, MINT_AMOUNT, logger);
     expect(ROUNDS).toBeLessThanOrEqual(MINT_AMOUNT);
   });
diff --git a/yarn-project/end-to-end/src/spartan/k8_utils.ts b/yarn-project/end-to-end/src/spartan/k8_utils.ts
new file mode 100644
index 00000000000..9fd2b81a827
--- /dev/null
+++ b/yarn-project/end-to-end/src/spartan/k8_utils.ts
@@ -0,0 +1,222 @@
+import { createDebugLogger } from '@aztec/aztec.js';
+
+import { exec, spawn } from 'child_process';
+import path from 'path';
+import { promisify } from 'util';
+import { z } from 'zod';
+
+const execAsync = promisify(exec);
+
+const logger = createDebugLogger('k8s-utils');
+
+const k8sConfigSchema = z.object({
+  NAMESPACE: z.string().min(1, 'NAMESPACE env variable must be set'),
+  HOST_PXE_PORT: z.coerce.number().min(1, 'HOST_PXE_PORT env variable must be set'),
+  CONTAINER_PXE_PORT: z.coerce.number().default(8080),
+  HOST_ETHEREUM_PORT: z.coerce.number().min(1, 'HOST_ETHEREUM_PORT env variable must be set'),
+  CONTAINER_ETHEREUM_PORT: z.coerce.number().default(8545),
+  SPARTAN_DIR: z.string().min(1, 'SPARTAN_DIR env variable must be set'),
+  K8S: z.literal('true'),
+});
+
+const directConfigSchema = z.object({
+  PXE_URL: z.string().url('PXE_URL must be a valid URL'),
+  ETHEREUM_HOST: z.string().url('ETHEREUM_HOST must be a valid URL'),
+  K8S: z.literal('false'),
+});
+
+const envSchema = z.discriminatedUnion('K8S', [k8sConfigSchema, directConfigSchema]);
+
+export type K8sConfig = z.infer<typeof k8sConfigSchema>;
+export type DirectConfig = z.infer<typeof directConfigSchema>;
+export type EnvConfig = z.infer<typeof envSchema>;
+
+export function getConfig(env: unknown): EnvConfig {
+  return envSchema.parse(env);
+}
+
+export function isK8sConfig(config: EnvConfig): config is K8sConfig {
+  return config.K8S === 'true';
+}
+
+export async function startPortForward({
+  resource,
+  namespace,
+  containerPort,
+  hostPort,
+}: {
+  resource: string;
+  namespace: string;
+  containerPort: number;
+  hostPort: number;
+}) {
+  // check if kubectl is already forwarding this port
+  try {
+    const command = `ps aux | grep 'kubectl.*${hostPort}:${containerPort}' | grep -v grep | awk '{print $2}'`;
+    const { stdout: processId } = await execAsync(command);
+    if (processId) {
+      logger.info(`Restarting port forward for ${resource}:${hostPort}`);
+      // kill the existing port forward
+      await execAsync(`kill -9 ${processId}`);
+    }
+  } catch (e) {
+    logger.info(`No existing port forward found for ${resource}:${hostPort}`);
+  }
+
+  logger.info(`kubectl port-forward -n ${namespace} ${resource} ${hostPort}:${containerPort}`);
+
+  const process = spawn('kubectl', ['port-forward', '-n', namespace, resource, `${hostPort}:${containerPort}`], {
+    detached: true,
+    windowsHide: true,
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+
+  process.stdout?.on('data', data => {
+    logger.info(data.toString());
+  });
+  process.stderr?.on('data', data => {
+    // It's a strange thing:
+    // If we don't pipe stderr, then the port forwarding does not work.
+    // Log to silent because this doesn't actually report errors,
+    // just extremely verbose debug logs.
+    logger.debug(data.toString());
+  });
+
+  // Wait a moment for the port forward to establish
+  await new Promise(resolve => setTimeout(resolve, 2000));
+
+  return process;
+}
+
+export async function deleteResourceByName({
+  resource,
+  namespace,
+  name,
+}: {
+  resource: string;
+  namespace: string;
+  name: string;
+}) {
+  const command = `kubectl delete ${resource} ${name} -n ${namespace} --ignore-not-found=true --wait=true`;
+  logger.info(`command: ${command}`);
+  const { stdout } = await execAsync(command);
+  return stdout;
+}
+
+export async function deleteResourceByLabel({
+  resource,
+  namespace,
+  label,
+}: {
+  resource: string;
+  namespace: string;
+  label: string;
+}) {
+  const command = `kubectl delete ${resource} -l ${label} -n ${namespace} --ignore-not-found=true --wait=true`;
+  logger.info(`command: ${command}`);
+  const { stdout } = await execAsync(command);
+  return stdout;
+}
+
+export async function waitForResourceByLabel({
+  resource,
+  label,
+  namespace,
+  condition = 'Ready',
+  timeout = '10m',
+}: {
+  resource: string;
+  label: string;
+  namespace: string;
+  condition?: string;
+  timeout?: string;
+}) {
+  const command = `kubectl wait ${resource} -l ${label} --for=condition=${condition} -n ${namespace} --timeout=${timeout}`;
+  logger.info(`command: ${command}`);
+  const { stdout } = await execAsync(command);
+  return stdout;
+}
+
+export function getChartDir(spartanDir: string, chartName: string) {
+  return path.join(spartanDir.trim(), chartName);
+}
+
+function valuesToArgs(values: Record<string, string>) {
+  return Object.entries(values)
+    .map(([key, value]) => `--set ${key}=${value}`)
+    .join(' ');
+}
+
+/**
+ * Installs a Helm chart with the given parameters.
+ * @param instanceName - The name of the Helm chart instance.
+ * @param targetNamespace - The namespace with the resources to be affected by the Helm chart.
+ * @param valuesFile - The values file to use for the Helm chart.
+ * @param chaosMeshNamespace - The namespace to install the Helm chart in.
+ * @param timeout - The timeout for the Helm command.
+ * @param clean - Whether to clean up the Helm chart before installing it.
+ * @returns The stdout of the Helm command.
+ * @throws If the Helm command fails.
+ *
+ * Example usage:
+ * ```typescript
+ * const stdout = await installChaosMeshChart({ instanceName: 'force-reorg', targetNamespace: 'smoke', valuesFile: 'kill-provers.yaml'});
+ * console.log(stdout);
+ * ```
+ */
+export async function installChaosMeshChart({
+  instanceName,
+  targetNamespace,
+  valuesFile,
+  helmChartDir,
+  chaosMeshNamespace = 'chaos-mesh',
+  timeout = '5m',
+  clean = true,
+  values = {},
+}: {
+  instanceName: string;
+  targetNamespace: string;
+  valuesFile: string;
+  helmChartDir: string;
+  chaosMeshNamespace?: string;
+  timeout?: string;
+  clean?: boolean;
+  values?: Record<string, string>;
+}) {
+  if (clean) {
+    // uninstall the helm chart if it exists
+    await execAsync(`helm uninstall ${instanceName} --namespace ${chaosMeshNamespace} --wait --ignore-not-found`);
+    // and delete the podchaos resource
+    await deleteResourceByName({
+      resource: 'podchaos',
+      namespace: chaosMeshNamespace,
+      name: `${targetNamespace}-${instanceName}`,
+    });
+  }
+
+  const helmCommand = `helm upgrade --install ${instanceName} ${helmChartDir} --namespace ${chaosMeshNamespace} --values ${helmChartDir}/values/${valuesFile} --wait --timeout=${timeout} --set global.targetNamespace=${targetNamespace}  ${valuesToArgs(
+    values,
+  )}`;
+  const { stdout } = await execAsync(helmCommand);
+  return stdout;
+}
+
+export function applyKillProvers({
+  namespace,
+  spartanDir,
+  durationSeconds,
+}: {
+  namespace: string;
+  spartanDir: string;
+  durationSeconds: number;
+}) {
+  return installChaosMeshChart({
+    instanceName: 'kill-provers',
+    targetNamespace: namespace,
+    valuesFile: 'kill-provers.yaml',
+    helmChartDir: getChartDir(spartanDir, 'network-shaping'),
+    values: {
+      'networkShaping.conditions.killProvers.duration': `${durationSeconds}s`,
+    },
+  });
+}
diff --git a/yarn-project/end-to-end/src/spartan/reorg.test.ts b/yarn-project/end-to-end/src/spartan/reorg.test.ts
new file mode 100644
index 00000000000..0847637b10a
--- /dev/null
+++ b/yarn-project/end-to-end/src/spartan/reorg.test.ts
@@ -0,0 +1,123 @@
+import { EthCheatCodes, sleep } from '@aztec/aztec.js';
+import { AZTEC_EPOCH_DURATION, AZTEC_SLOT_DURATION } from '@aztec/circuits.js';
+import { createDebugLogger } from '@aztec/foundation/log';
+
+import { expect, jest } from '@jest/globals';
+
+import { RollupCheatCodes } from '../../../aztec.js/src/utils/cheat_codes.js';
+import {
+  applyKillProvers,
+  deleteResourceByLabel,
+  getConfig,
+  isK8sConfig,
+  startPortForward,
+  waitForResourceByLabel,
+} from './k8_utils.js';
+import { type TestWallets, performTransfers, setupTestWalletsWithTokens } from './setup_test_wallets.js';
+
+const config = getConfig(process.env);
+if (!isK8sConfig(config)) {
+  throw new Error('This test must be run in a k8s environment');
+}
+const { NAMESPACE, HOST_PXE_PORT, HOST_ETHEREUM_PORT, CONTAINER_PXE_PORT, CONTAINER_ETHEREUM_PORT, SPARTAN_DIR } =
+  config;
+const debugLogger = createDebugLogger('aztec:spartan-test:reorg');
+
+async function checkBalances(testWallets: TestWallets, mintAmount: bigint, totalAmountTransferred: bigint) {
+  testWallets.wallets.forEach(async w => {
+    expect(await testWallets.tokenAdminWallet.methods.balance_of_public(w.getAddress()).simulate()).toBe(
+      mintAmount - totalAmountTransferred,
+    );
+  });
+
+  expect(
+    await testWallets.tokenAdminWallet.methods.balance_of_public(testWallets.recipientWallet.getAddress()).simulate(),
+  ).toBe(totalAmountTransferred * BigInt(testWallets.wallets.length));
+}
+
+describe('reorg test', () => {
+  jest.setTimeout(60 * 60 * 1000); // 60 minutes
+
+  const MINT_AMOUNT = 2_000_000n;
+  const SETUP_EPOCHS = 2;
+  const TRANSFER_AMOUNT = 1n;
+  const ETHEREUM_HOST = `http://127.0.0.1:${HOST_ETHEREUM_PORT}`;
+  const PXE_URL = `http://127.0.0.1:${HOST_PXE_PORT}`;
+
+  let testWallets: TestWallets;
+
+  it('survives a reorg', async () => {
+    await startPortForward({
+      resource: 'svc/spartan-aztec-network-pxe',
+      namespace: NAMESPACE,
+      containerPort: CONTAINER_PXE_PORT,
+      hostPort: HOST_PXE_PORT,
+    });
+    await startPortForward({
+      resource: 'svc/spartan-aztec-network-ethereum',
+      namespace: NAMESPACE,
+      containerPort: CONTAINER_ETHEREUM_PORT,
+      hostPort: HOST_ETHEREUM_PORT,
+    });
+    testWallets = await setupTestWalletsWithTokens(PXE_URL, MINT_AMOUNT, debugLogger);
+    const ethCheatCodes = new EthCheatCodes(ETHEREUM_HOST);
+    const rollupCheatCodes = new RollupCheatCodes(
+      ethCheatCodes,
+      await testWallets.pxe.getNodeInfo().then(n => n.l1ContractAddresses),
+    );
+
+    await performTransfers({
+      testWallets,
+      rounds: AZTEC_EPOCH_DURATION * SETUP_EPOCHS,
+      transferAmount: TRANSFER_AMOUNT,
+      logger: debugLogger,
+    });
+    await checkBalances(testWallets, MINT_AMOUNT, TRANSFER_AMOUNT * BigInt(AZTEC_EPOCH_DURATION * SETUP_EPOCHS));
+
+    // get the tips before the reorg
+    const { pending: preReorgPending, proven: preReorgProven } = await rollupCheatCodes.getTips();
+
+    // kill the provers
+    const stdout = await applyKillProvers({
+      namespace: NAMESPACE,
+      spartanDir: SPARTAN_DIR,
+      durationSeconds: AZTEC_EPOCH_DURATION * AZTEC_SLOT_DURATION * 2,
+    });
+    debugLogger.info(stdout);
+
+    // We only need 2 epochs for a reorg to be triggered, but 3 gives time for the bot to be restarted and the chain to re-stabilize
+    // TODO(#9613): why do we need to wait for 2 epochs?
+    debugLogger.info(`Waiting for 2 epochs to pass`);
+    await sleep(AZTEC_EPOCH_DURATION * AZTEC_SLOT_DURATION * 2 * 1000);
+
+    // TODO(#9327): begin delete
+    // The bot must be restarted because the PXE does not handle reorgs without a restart.
+    // When the issue is fixed, we can remove the following delete, wait, startPortForward, and setupTestWallets
+    await deleteResourceByLabel({ resource: 'pods', namespace: NAMESPACE, label: 'app=pxe' });
+    await sleep(30 * 1000);
+    await waitForResourceByLabel({ resource: 'pods', namespace: NAMESPACE, label: 'app=pxe' });
+    await sleep(30 * 1000);
+    await startPortForward({
+      resource: 'svc/spartan-aztec-network-pxe',
+      namespace: NAMESPACE,
+      containerPort: CONTAINER_PXE_PORT,
+      hostPort: HOST_PXE_PORT,
+    });
+    testWallets = await setupTestWalletsWithTokens(PXE_URL, MINT_AMOUNT, debugLogger);
+    // TODO(#9327): end delete
+
+    await performTransfers({
+      testWallets,
+      rounds: AZTEC_EPOCH_DURATION * SETUP_EPOCHS,
+      transferAmount: TRANSFER_AMOUNT,
+      logger: debugLogger,
+    });
+
+    // expect the block height to be at least 4 epochs worth of slots
+    const { pending: newPending, proven: newProven } = await rollupCheatCodes.getTips();
+    expect(newPending).toBeGreaterThan(preReorgPending);
+    expect(newPending).toBeGreaterThan(4 * AZTEC_EPOCH_DURATION);
+    expect(newProven).toBeGreaterThan(preReorgProven);
+    expect(newProven).toBeGreaterThan(3 * AZTEC_EPOCH_DURATION);
+  });
+});
diff --git a/yarn-project/end-to-end/src/spartan/setup_test_wallets.ts b/yarn-project/end-to-end/src/spartan/setup_test_wallets.ts
index 99121d5f664..5feb52d1de8 100644
--- a/yarn-project/end-to-end/src/spartan/setup_test_wallets.ts
+++ b/yarn-project/end-to-end/src/spartan/setup_test_wallets.ts
@@ -76,3 +76,33 @@ export async function setupTestWalletsWithTokens(
 
   return { pxe, wallets, tokenAdminWallet, tokenName: TOKEN_NAME, tokenAddress, recipientWallet };
 }
+
+export async function performTransfers({
+  testWallets,
+  rounds,
+  transferAmount,
+  logger,
+}: {
+  testWallets: TestWallets;
+  rounds: number;
+  transferAmount: bigint;
+  logger: Logger;
+}) {
+  const recipient = testWallets.recipientWallet.getAddress();
+
+  for (let i = 0; i < rounds; i++) {
+    const interactions = await Promise.all(
+      testWallets.wallets.map(async w =>
+        (
+          await TokenContract.at(testWallets.tokenAddress, w)
+        ).methods.transfer_public(w.getAddress(), recipient, transferAmount, 0),
+      ),
+    );
+
+    const txs = await Promise.all(interactions.map(async i => await i.prove()));
+
+    await Promise.all(txs.map(t => t.send().wait({ timeout: 600 })));
+
+    logger.info(`Completed round ${i + 1} / ${rounds}`);
+  }
+}
diff --git a/yarn-project/end-to-end/src/spartan/smoke.test.ts b/yarn-project/end-to-end/src/spartan/smoke.test.ts
index 8f270f53407..4eafdd9d4ef 100644
--- a/yarn-project/end-to-end/src/spartan/smoke.test.ts
+++ b/yarn-project/end-to-end/src/spartan/smoke.test.ts
@@ -5,16 +5,28 @@ import { RollupAbi } from '@aztec/l1-artifacts';
 import { createPublicClient, getAddress, getContract, http } from 'viem';
 import { foundry } from 'viem/chains';
 
-const { PXE_URL } = process.env;
-if (!PXE_URL) {
-  throw new Error('PXE_URL env variable must be set');
-}
+import { getConfig, isK8sConfig, startPortForward } from './k8_utils.js';
+
+const config = getConfig(process.env);
+
 const debugLogger = createDebugLogger('aztec:spartan-test:smoke');
 // const userLog = createConsoleLogger();
 
 describe('smoke test', () => {
   let pxe: PXE;
   beforeAll(async () => {
+    let PXE_URL;
+    if (isK8sConfig(config)) {
+      await startPortForward({
+        resource: 'svc/spartan-aztec-network-pxe',
+        namespace: config.NAMESPACE,
+        containerPort: config.CONTAINER_PXE_PORT,
+        hostPort: config.HOST_PXE_PORT,
+      });
+      PXE_URL = `http://127.0.0.1:${config.HOST_PXE_PORT}`;
+    } else {
+      PXE_URL = config.PXE_URL;
+    }
     pxe = await createCompatibleClient(PXE_URL, debugLogger);
   });
   it('should be able to get node enr', async () => {
diff --git a/yarn-project/end-to-end/src/spartan/transfer.test.ts b/yarn-project/end-to-end/src/spartan/transfer.test.ts
index ebbc84b7069..2988c23b9cb 100644
--- a/yarn-project/end-to-end/src/spartan/transfer.test.ts
+++ b/yarn-project/end-to-end/src/spartan/transfer.test.ts
@@ -4,12 +4,10 @@ import { TokenContract } from '@aztec/noir-contracts.js';
 
 import { jest } from '@jest/globals';
 
+import { getConfig, isK8sConfig, startPortForward } from './k8_utils.js';
 import { type TestWallets, setupTestWalletsWithTokens } from './setup_test_wallets.js';
 
-const { PXE_URL } = process.env;
-if (!PXE_URL) {
-  throw new Error('PXE_URL env variable must be set');
-}
+const config = getConfig(process.env);
 
 describe('token transfer test', () => {
   jest.setTimeout(10 * 60 * 2000); // 20 minutes
@@ -22,6 +20,18 @@ describe('token transfer test', () => {
   let testWallets: TestWallets;
 
   beforeAll(async () => {
+    let PXE_URL;
+    if (isK8sConfig(config)) {
+      await startPortForward({
+        resource: 'svc/spartan-aztec-network-pxe',
+        namespace: config.NAMESPACE,
+        containerPort: config.CONTAINER_PXE_PORT,
+        hostPort: config.HOST_PXE_PORT,
+      });
+      PXE_URL = `http://127.0.0.1:${config.HOST_PXE_PORT}`;
+    } else {
+      PXE_URL = config.PXE_URL;
+    }
     testWallets = await setupTestWalletsWithTokens(PXE_URL, MINT_AMOUNT, logger);
     expect(ROUNDS).toBeLessThanOrEqual(MINT_AMOUNT);
   });
diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock
index d31f814b645..7d3ed0c0c24 100644
--- a/yarn-project/yarn.lock
+++ b/yarn-project/yarn.lock
@@ -584,6 +584,7 @@ __metadata:
     webpack: ^5.88.2
     webpack-cli: ^5.1.4
     winston: ^3.10.0
+    zod: ^3.23.8
   languageName: unknown
   linkType: soft
 
@@ -16757,7 +16758,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"zod@npm:^3.22.4":
+"zod@npm:^3.22.4, zod@npm:^3.23.8":
   version: 3.23.8
   resolution: "zod@npm:3.23.8"
   checksum: 15949ff82118f59c893dacd9d3c766d02b6fa2e71cf474d5aa888570c469dbf5446ac5ad562bb035bf7ac9650da94f290655c194f4a6de3e766f43febd432c5c

From 1b87ef48f683f97678cad148fdf3774b9f417c4b Mon Sep 17 00:00:00 2001
From: Mitch <mitchell@aztecprotocol.com>
Date: Thu, 31 Oct 2024 13:25:37 -0400
Subject: [PATCH 2/6] bump the sleep back to 3 epochs.

---
 yarn-project/end-to-end/src/spartan/reorg.test.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/yarn-project/end-to-end/src/spartan/reorg.test.ts b/yarn-project/end-to-end/src/spartan/reorg.test.ts
index 0847637b10a..d3a591c9cb0 100644
--- a/yarn-project/end-to-end/src/spartan/reorg.test.ts
+++ b/yarn-project/end-to-end/src/spartan/reorg.test.ts
@@ -86,9 +86,9 @@ describe('reorg test', () => {
     debugLogger.info(stdout);
 
     // We only need 2 epochs for a reorg to be triggered, but 3 gives time for the bot to be restarted and the chain to re-stabilize
-    // TODO(#9613): why do we need to wait for 2 epochs?
-    debugLogger.info(`Waiting for 2 epochs to pass`);
-    await sleep(AZTEC_EPOCH_DURATION * AZTEC_SLOT_DURATION * 2 * 1000);
+    // TODO(#9613): why do we need to wait for 3 epochs?
+    debugLogger.info(`Waiting for 3 epochs to pass`);
+    await sleep(AZTEC_EPOCH_DURATION * AZTEC_SLOT_DURATION * 3 * 1000);
 
     // TODO(#9327): begin delete
     // The bot must be restarted because the PXE does not handle reorgs without a restart.

From 2fb1c8b72ac7d79355924a0234fcdb754c213277 Mon Sep 17 00:00:00 2001
From: Mitch <mitchell@aztecprotocol.com>
Date: Thu, 31 Oct 2024 18:32:10 -0400
Subject: [PATCH 3/6] reduce master tests to 3 validators

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0563cfe90e8..6f569ceb13d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -691,7 +691,7 @@ jobs:
             cd yarn-project/end-to-end
             echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
             test=${{ matrix.test }}
-            NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=16-validators.yaml ./scripts/network_test.sh ./src/spartan/$test
+            NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=3-validators.yaml ./scripts/network_test.sh ./src/spartan/$test
 
   l1-contracts-test:
     needs: [build, configure]

From 05d9b774e4df9aa4f221d6c612977b62de30c398 Mon Sep 17 00:00:00 2001
From: Mitch <mitchell@aztecprotocol.com>
Date: Thu, 31 Oct 2024 20:18:34 -0400
Subject: [PATCH 4/6] turn back down the tests on master

---
 .github/workflows/ci.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6f569ceb13d..e122b4b67cd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -664,7 +664,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: [smoke.test.ts, transfer.test.ts, reorg.test.ts]
+        # TODO(#9640): add transfer.test.ts and reorg.test.ts
+        test: [smoke.test.ts]
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -687,11 +688,12 @@ jobs:
           # command to produce the images in case they don't exist
           builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
           tester_ttl: 60
+          # TODO(#9640): remove `|| true` and use 16-validators.yaml
           run: |
             cd yarn-project/end-to-end
             echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
             test=${{ matrix.test }}
-            NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=3-validators.yaml ./scripts/network_test.sh ./src/spartan/$test
+            NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=default.yaml ./scripts/network_test.sh ./src/spartan/$test || true
 
   l1-contracts-test:
     needs: [build, configure]

From 44fb2922b4dac7a0f50383ee8edd60d5fe14efd2 Mon Sep 17 00:00:00 2001
From: Mitch <mitchell@aztecprotocol.com>
Date: Thu, 31 Oct 2024 21:10:04 -0400
Subject: [PATCH 5/6] no kind tests on master

---
 .github/workflows/ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e122b4b67cd..7c5b83c2a6e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -659,7 +659,8 @@ jobs:
   # note: proving disabled
   kind-network-test:
     needs: [build, configure]
-    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master'
+    # TODO(#9640): remove this once the test is re-enabled on master
+    if: github.ref_name == '9640-re-enable-kind-network-test-on-master'
     runs-on: ${{ needs.configure.outputs.username }}-x86
     strategy:
       fail-fast: false

From d72e275d0abfdff83271c953a930dcd26ec287a4 Mon Sep 17 00:00:00 2001
From: Mitch <mitchell@aztecprotocol.com>
Date: Thu, 31 Oct 2024 21:29:56 -0400
Subject: [PATCH 6/6] disable kind tests on master

---
 .github/workflows/ci.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7c5b83c2a6e..e1cffa2732a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -659,8 +659,7 @@ jobs:
   # note: proving disabled
   kind-network-test:
     needs: [build, configure]
-    # TODO(#9640): remove this once the test is re-enabled on master
-    if: github.ref_name == '9640-re-enable-kind-network-test-on-master'
+    if: needs.configure.outputs.yarn-project == 'true' || github.ref_name == 'master'
     runs-on: ${{ needs.configure.outputs.username }}-x86
     strategy:
       fail-fast: false
@@ -676,7 +675,7 @@ jobs:
       - name: Setup and KIND Network Test
         timeout-minutes: 60
         uses: ./.github/ensure-tester-with-images
-        if: matrix.test == 'smoke.test.ts' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'network-all')
+        if: contains(github.event.pull_request.labels.*.name, 'kind-network-all')
         env:
           USERNAME: ${{ needs.configure.outputs.username }}
         with: