feat: test release network via ci workflow (#10388)

New workflow to run an arbitrary test on an arbitrary namespace (within
our aztec-gke cluster) with an arbitrary e2e image.

I tested this with a temporary `on: push` flag. See successful run
[here](https://github.com/AztecProtocol/aztec-packages/actions/runs/12148432809/job/33876880156)
using smoke test with a locally built image against the smoke cluster.

Also tweak our release to also publish our `end-to-end` image so it can
be used by this workflow more easily.

Fix #10247 
Fix #10383

.github/workflows/network-test.yml

@@ -0,0 +1,86 @@
+name: Aztec Network Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      namespace:
+        description: The namespace to deploy to, e.g. smoke
+        required: true
+      test:
+        description: The test to run, e.g. spartan/smoke.test.ts
+        required: true
+      aztec_e2e_docker_image:
+        description: The Aztec E2E Docker image to use, e.g. aztecprotocol/end-to-end:da809c58290f9590836f45ec59376cbf04d3c4ce-x86_64
+        required: true
+
+jobs:
+  network_test:
+    runs-on: ubuntu-latest
+
+    env:
+      TEST_DOCKER_IMAGE: ${{ inputs.aztec_e2e_docker_image }}
+      NAMESPACE: ${{ inputs.namespace }}
+      TEST: ${{ inputs.test }}
+      CHART_PATH: ./spartan/aztec-network
+      CLUSTER_NAME: aztec-gke
+      REGION: us-west1-a
+      PROJECT_ID: testnet-440309
+      GKE_CLUSTER_CONTEXT: gke_testnet-440309_us-west1-a_aztec-gke
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_SA_KEY }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          install_components: gke-gcloud-auth-plugin
+
+      - name: Configure kubectl with GKE cluster
+        run: |
+          gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --region ${{ env.REGION }}
+
+      - name: Run test
+        run: |
+
+          # Find 3 free ports between 9000 and 10000
+          FREE_PORTS=$(comm -23 <(seq 9000 10000 | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 3)
+
+          # Extract the free ports from the list
+          PXE_PORT=$(echo $FREE_PORTS | awk '{print $1}')
+          ANVIL_PORT=$(echo $FREE_PORTS | awk '{print $2}')
+          METRICS_PORT=$(echo $FREE_PORTS | awk '{print $3}')
+
+          export GRAFANA_PASSWORD=$(kubectl get secrets -n metrics metrics-grafana -o jsonpath='{.data.admin-password}' | base64 --decode)
+
+          gcloud config set project ${{ env.PROJECT_ID }}
+
+          GCLOUD_CONFIG_DIR=$(gcloud info --format='value(config. paths. global_config_dir)')
+
+          echo "gcloud config dir: [$GCLOUD_CONFIG_DIR]"
+
+          docker run --rm --network=host \
+            -v ~/.kube:/root/.kube \
+            -v $GCLOUD_CONFIG_DIR:/root/.config/gcloud \
+            -e K8S=gcloud \
+            -e CLUSTER_NAME=${{ env.CLUSTER_NAME }} \
+            -e REGION=${{ env.REGION }} \
+            -e INSTANCE_NAME=${{ env.NAMESPACE }} \
+            -e SPARTAN_DIR="/usr/src/spartan" \
+            -e NAMESPACE=${{ env.NAMESPACE }} \
+            -e HOST_PXE_PORT=$PXE_PORT \
+            -e CONTAINER_PXE_PORT=8081 \
+            -e HOST_ETHEREUM_PORT=$ANVIL_PORT \
+            -e CONTAINER_ETHEREUM_PORT=8545 \
+            -e HOST_METRICS_PORT=$METRICS_PORT \
+            -e CONTAINER_METRICS_PORT=80 \
+            -e GRAFANA_PASSWORD=$GRAFANA_PASSWORD \
+            -e DEBUG="aztec:*" \
+            -e LOG_JSON=1 \
+            -e LOG_LEVEL=debug \
+            ${{ env.TEST_DOCKER_IMAGE }} ${{ env.TEST }}

.github/workflows/publish-aztec-packages.yml

@@ -100,13 +100,13 @@ jobs:
         with:
           concurrency_key: build-aztec
           dockerhub_password: "${{ env.DOCKERHUB_PASSWORD }}"
-      - name: Build & Push Aztec x86_64
+      - name: Build & Push Aztec and End-to-End x86_64
         timeout-minutes: 40
         run: |
           earthly-ci \
             --no-output \
             --push \
-            ./yarn-project+export-aztec-arch \
+            ./yarn-project+export-images-arch \
             --DIST_TAG=${{ env.GIT_COMMIT }} \
             --ARCH=x86_64
 

yarn-project/Earthfile

@@ -215,11 +215,17 @@ end-to-end-base:
         && apt-get install -y wget gnupg \
         && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
         && echo "deb [arch=$(dpkg --print-architecture)] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \
-        && apt update && apt install curl chromium nodejs netcat-openbsd -y \
+        && apt update && apt install curl chromium nodejs netcat-openbsd git -y \
         && rm -rf /var/lib/apt/lists/* \
         && mkdir -p /usr/local/bin \
         && curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
         && chmod +x /usr/local/bin/kubectl \
+        && curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-linux-x86_64.tar.gz \
+        && tar xf google-cloud-cli-linux-x86_64.tar.gz \
+        && mv google-cloud-sdk /usr/lib/google-cloud-sdk \
+        && /usr/lib/google-cloud-sdk/install.sh --additional-components gke-gcloud-auth-plugin --path-update false --quiet \
+        && ln -s /usr/lib/google-cloud-sdk/bin/gcloud /usr/bin/gcloud \
+        && rm google-cloud-cli-linux-x86_64.tar.gz \
         && curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 \
         && chmod +x get_helm.sh \
         && ./get_helm.sh \
@@ -270,10 +276,22 @@ export-end-to-end:
     FROM +end-to-end
     SAVE IMAGE aztecprotocol/end-to-end:$EARTHLY_GIT_HASH
 
+export-end-to-end-arch:
+    FROM +end-to-end
+    ARG DIST_TAG="latest"
+    ARG ARCH
+    SAVE IMAGE --push aztecprotocol/end-to-end:${DIST_TAG}${ARCH:+-$ARCH}
+
 export-e2e-test-images:
     BUILD +export-aztec
     BUILD +export-end-to-end
 
+export-images-arch:
+    ARG DIST_TAG="latest"
+    ARG ARCH
+    BUILD +export-aztec-arch
+    BUILD +export-end-to-end-arch
+
 format-check:
     FROM +build
     RUN yarn formatting

yarn-project/end-to-end/scripts/network_test.sh

@@ -142,7 +142,7 @@ kubectl wait pod -l app==pxe --for=condition=Ready -n "$NAMESPACE" --timeout=10m
 # Find 3 free ports between 9000 and 10000
 FREE_PORTS=$(comm -23 <(seq 9000 10000 | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 3)
 
-# Extract the two free ports from the list
+# Extract the free ports from the list
 PXE_PORT=$(echo $FREE_PORTS | awk '{print $1}')
 ANVIL_PORT=$(echo $FREE_PORTS | awk '{print $2}')
 METRICS_PORT=$(echo $FREE_PORTS | awk '{print $3}')
@@ -165,7 +165,7 @@ fi
 
 docker run --rm --network=host \
   -v ~/.kube:/root/.kube \
-  -e K8S=true \
+  -e K8S=local \
   -e INSTANCE_NAME="spartan" \
   -e SPARTAN_DIR="/usr/src/spartan" \
   -e NAMESPACE="$NAMESPACE" \

yarn-project/end-to-end/src/spartan/4epochs.test.ts

@@ -7,9 +7,9 @@ import { jest } from '@jest/globals';
 
 import { RollupCheatCodes } from '../../../aztec.js/src/utils/cheat_codes.js';
 import { type TestWallets, setupTestWalletsWithTokens } from './setup_test_wallets.js';
-import { getConfig, isK8sConfig, startPortForward } from './utils.js';
+import { isK8sConfig, setupEnvironment, startPortForward } from './utils.js';
 
-const config = getConfig(process.env);
+const config = setupEnvironment(process.env);
 
 describe('token transfer test', () => {
   jest.setTimeout(10 * 60 * 4000); // 40 minutes

yarn-project/end-to-end/src/spartan/gating-passive.test.ts

@@ -11,10 +11,10 @@ import {
   applyValidatorKill,
   awaitL2BlockNumber,
   enableValidatorDynamicBootNode,
-  getConfig,
   isK8sConfig,
   restartBot,
   runAlertCheck,
+  setupEnvironment,
   startPortForward,
 } from './utils.js';
 
@@ -28,7 +28,7 @@ const qosAlerts: AlertConfig[] = [
   },
 ];
 
-const config = getConfig(process.env);
+const config = setupEnvironment(process.env);
 if (!isK8sConfig(config)) {
   throw new Error('This test must be run in a k8s environment');
 }

yarn-project/end-to-end/src/spartan/proving.test.ts

@@ -4,11 +4,11 @@ import { createDebugLogger } from '@aztec/foundation/log';
 import { jest } from '@jest/globals';
 import { type ChildProcess } from 'child_process';
 
-import { getConfig, isK8sConfig, startPortForward } from './utils.js';
+import { isK8sConfig, setupEnvironment, startPortForward } from './utils.js';
 
 jest.setTimeout(2_400_000); // 40 minutes
 
-const config = getConfig(process.env);
+const config = setupEnvironment(process.env);
 const debugLogger = createDebugLogger('aztec:spartan-test:proving');
 const SLEEP_MS = 1000;
 

yarn-project/end-to-end/src/spartan/reorg.test.ts

@@ -8,13 +8,13 @@ import { type TestWallets, performTransfers, setupTestWalletsWithTokens } from '
 import {
   applyProverFailure,
   deleteResourceByLabel,
-  getConfig,
   isK8sConfig,
+  setupEnvironment,
   startPortForward,
   waitForResourceByLabel,
 } from './utils.js';
 
-const config = getConfig(process.env);
+const config = setupEnvironment(process.env);
 if (!isK8sConfig(config)) {
   throw new Error('This test must be run in a k8s environment');
 }

yarn-project/end-to-end/src/spartan/smoke.test.ts

@@ -6,9 +6,9 @@ import { createPublicClient, getAddress, getContract, http } from 'viem';
 import { foundry } from 'viem/chains';
 
 import { type AlertConfig } from '../quality_of_service/alert_checker.js';
-import { getConfig, isK8sConfig, runAlertCheck, startPortForward } from './utils.js';
+import { isK8sConfig, runAlertCheck, setupEnvironment, startPortForward } from './utils.js';
 
-const config = getConfig(process.env);
+const config = setupEnvironment(process.env);
 
 const debugLogger = createDebugLogger('aztec:spartan-test:smoke');
 

yarn-project/end-to-end/src/spartan/transfer.test.ts

@@ -5,9 +5,9 @@ import { TokenContract } from '@aztec/noir-contracts.js';
 import { jest } from '@jest/globals';
 
 import { type TestWallets, setupTestWalletsWithTokens } from './setup_test_wallets.js';
-import { getConfig, isK8sConfig, startPortForward } from './utils.js';
+import { isK8sConfig, setupEnvironment, startPortForward } from './utils.js';
 
-const config = getConfig(process.env);
+const config = setupEnvironment(process.env);
 
 describe('token transfer test', () => {
   jest.setTimeout(10 * 60 * 2000); // 20 minutes

yarn-project/end-to-end/src/spartan/utils.ts

@@ -1,7 +1,7 @@
 import { createDebugLogger, sleep } from '@aztec/aztec.js';
 import type { Logger } from '@aztec/foundation/log';
 
-import { exec, spawn } from 'child_process';
+import { exec, execSync, spawn } from 'child_process';
 import path from 'path';
 import { promisify } from 'util';
 import { z } from 'zod';
@@ -13,7 +13,7 @@ const execAsync = promisify(exec);
 
 const logger = createDebugLogger('k8s-utils');
 
-const k8sConfigSchema = z.object({
+const k8sLocalConfigSchema = z.object({
   INSTANCE_NAME: z.string().min(1, 'INSTANCE_NAME env variable must be set'),
   NAMESPACE: z.string().min(1, 'NAMESPACE env variable must be set'),
   HOST_PXE_PORT: z.coerce.number().min(1, 'HOST_PXE_PORT env variable must be set'),
@@ -25,7 +25,13 @@ const k8sConfigSchema = z.object({
   GRAFANA_PASSWORD: z.string().min(1, 'GRAFANA_PASSWORD env variable must be set'),
   METRICS_API_PATH: z.string().default('/api/datasources/proxy/uid/spartan-metrics-prometheus/api/v1/query'),
   SPARTAN_DIR: z.string().min(1, 'SPARTAN_DIR env variable must be set'),
-  K8S: z.literal('true'),
+  K8S: z.literal('local'),
+});
+
+const k8sGCloudConfigSchema = k8sLocalConfigSchema.extend({
+  K8S: z.literal('gcloud'),
+  CLUSTER_NAME: z.string().min(1, 'CLUSTER_NAME env variable must be set'),
+  REGION: z.string().min(1, 'REGION env variable must be set'),
 });
 
 const directConfigSchema = z.object({
@@ -34,18 +40,28 @@ const directConfigSchema = z.object({
   K8S: z.literal('false'),
 });
 
-const envSchema = z.discriminatedUnion('K8S', [k8sConfigSchema, directConfigSchema]);
+const envSchema = z.discriminatedUnion('K8S', [k8sLocalConfigSchema, k8sGCloudConfigSchema, directConfigSchema]);
 
-export type K8sConfig = z.infer<typeof k8sConfigSchema>;
+export type K8sLocalConfig = z.infer<typeof k8sLocalConfigSchema>;
+export type K8sGCloudConfig = z.infer<typeof k8sGCloudConfigSchema>;
 export type DirectConfig = z.infer<typeof directConfigSchema>;
 export type EnvConfig = z.infer<typeof envSchema>;
 
-export function getConfig(env: unknown): EnvConfig {
-  return envSchema.parse(env);
+export function isK8sConfig(config: EnvConfig): config is K8sLocalConfig | K8sGCloudConfig {
+  return config.K8S === 'local' || config.K8S === 'gcloud';
 }
 
-export function isK8sConfig(config: EnvConfig): config is K8sConfig {
-  return config.K8S === 'true';
+export function isGCloudConfig(config: EnvConfig): config is K8sGCloudConfig {
+  return config.K8S === 'gcloud';
+}
+
+export function setupEnvironment(env: unknown): EnvConfig {
+  const config = envSchema.parse(env);
+  if (isGCloudConfig(config)) {
+    const command = `gcloud container clusters get-credentials ${config.CLUSTER_NAME} --region=${config.REGION}`;
+    execSync(command);
+  }
+  return config;
 }
 
 export async function startPortForward({