From e6060ecca318ca4cdc60f1df77c1e7639a745f79 Mon Sep 17 00:00:00 2001 From: just-mitch <68168980+just-mitch@users.noreply.github.com> Date: Tue, 3 Dec 2024 18:04:32 -0500 Subject: [PATCH] feat: test release network via ci workflow (#10388) New workflow to run an arbitrary test on an arbitrary namespace (within our aztec-gke cluster) with an arbitrary e2e image. I tested this with a temporary `on: push` flag. See successful run [here](https://github.com/AztecProtocol/aztec-packages/actions/runs/12148432809/job/33876880156) using smoke test with a locally built image against the smoke cluster. Also tweak our release to also publish our `end-to-end` image so it can be used by this workflow more easily. Fix #10247 Fix #10383 --- .github/workflows/network-test.yml | 86 +++++++++++++++++++ .github/workflows/publish-aztec-packages.yml | 4 +- yarn-project/Earthfile | 20 ++++- .../end-to-end/scripts/network_test.sh | 4 +- .../end-to-end/src/spartan/4epochs.test.ts | 4 +- .../src/spartan/gating-passive.test.ts | 4 +- .../end-to-end/src/spartan/proving.test.ts | 4 +- .../end-to-end/src/spartan/reorg.test.ts | 4 +- .../end-to-end/src/spartan/smoke.test.ts | 4 +- .../end-to-end/src/spartan/transfer.test.ts | 4 +- yarn-project/end-to-end/src/spartan/utils.ts | 34 ++++++-- 11 files changed, 146 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/network-test.yml diff --git a/.github/workflows/network-test.yml b/.github/workflows/network-test.yml new file mode 100644 index 00000000000..8ed37dfa51d --- /dev/null +++ b/.github/workflows/network-test.yml @@ -0,0 +1,86 @@ +name: Aztec Network Test + +on: + workflow_dispatch: + inputs: + namespace: + description: The namespace to deploy to, e.g. smoke + required: true + test: + description: The test to run, e.g. spartan/smoke.test.ts + required: true + aztec_e2e_docker_image: + description: The Aztec E2E Docker image to use, e.g. aztecprotocol/end-to-end:da809c58290f9590836f45ec59376cbf04d3c4ce-x86_64 + required: true + +jobs: + network_test: + runs-on: ubuntu-latest + + env: + TEST_DOCKER_IMAGE: ${{ inputs.aztec_e2e_docker_image }} + NAMESPACE: ${{ inputs.namespace }} + TEST: ${{ inputs.test }} + CHART_PATH: ./spartan/aztec-network + CLUSTER_NAME: aztec-gke + REGION: us-west1-a + PROJECT_ID: testnet-440309 + GKE_CLUSTER_CONTEXT: gke_testnet-440309_us-west1-a_aztec-gke + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + with: + install_components: gke-gcloud-auth-plugin + + - name: Configure kubectl with GKE cluster + run: | + gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --region ${{ env.REGION }} + + - name: Run test + run: | + + # Find 3 free ports between 9000 and 10000 + FREE_PORTS=$(comm -23 <(seq 9000 10000 | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 3) + + # Extract the free ports from the list + PXE_PORT=$(echo $FREE_PORTS | awk '{print $1}') + ANVIL_PORT=$(echo $FREE_PORTS | awk '{print $2}') + METRICS_PORT=$(echo $FREE_PORTS | awk '{print $3}') + + export GRAFANA_PASSWORD=$(kubectl get secrets -n metrics metrics-grafana -o jsonpath='{.data.admin-password}' | base64 --decode) + + gcloud config set project ${{ env.PROJECT_ID }} + + GCLOUD_CONFIG_DIR=$(gcloud info --format='value(config. paths. global_config_dir)') + + echo "gcloud config dir: [$GCLOUD_CONFIG_DIR]" + + docker run --rm --network=host \ + -v ~/.kube:/root/.kube \ + -v $GCLOUD_CONFIG_DIR:/root/.config/gcloud \ + -e K8S=gcloud \ + -e CLUSTER_NAME=${{ env.CLUSTER_NAME }} \ + -e REGION=${{ env.REGION }} \ + -e INSTANCE_NAME=${{ env.NAMESPACE }} \ + -e SPARTAN_DIR="/usr/src/spartan" \ + -e NAMESPACE=${{ env.NAMESPACE }} \ + -e HOST_PXE_PORT=$PXE_PORT \ + -e CONTAINER_PXE_PORT=8081 \ + -e HOST_ETHEREUM_PORT=$ANVIL_PORT \ + -e CONTAINER_ETHEREUM_PORT=8545 \ + -e HOST_METRICS_PORT=$METRICS_PORT \ + -e CONTAINER_METRICS_PORT=80 \ + -e GRAFANA_PASSWORD=$GRAFANA_PASSWORD \ + -e DEBUG="aztec:*" \ + -e LOG_JSON=1 \ + -e LOG_LEVEL=debug \ + ${{ env.TEST_DOCKER_IMAGE }} ${{ env.TEST }} diff --git a/.github/workflows/publish-aztec-packages.yml b/.github/workflows/publish-aztec-packages.yml index d28c577f875..2695f252c68 100644 --- a/.github/workflows/publish-aztec-packages.yml +++ b/.github/workflows/publish-aztec-packages.yml @@ -100,13 +100,13 @@ jobs: with: concurrency_key: build-aztec dockerhub_password: "${{ env.DOCKERHUB_PASSWORD }}" - - name: Build & Push Aztec x86_64 + - name: Build & Push Aztec and End-to-End x86_64 timeout-minutes: 40 run: | earthly-ci \ --no-output \ --push \ - ./yarn-project+export-aztec-arch \ + ./yarn-project+export-images-arch \ --DIST_TAG=${{ env.GIT_COMMIT }} \ --ARCH=x86_64 diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index 7fd30660cfd..c25214fc97a 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -215,11 +215,17 @@ end-to-end-base: && apt-get install -y wget gnupg \ && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ && echo "deb [arch=$(dpkg --print-architecture)] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \ - && apt update && apt install curl chromium nodejs netcat-openbsd -y \ + && apt update && apt install curl chromium nodejs netcat-openbsd git -y \ && rm -rf /var/lib/apt/lists/* \ && mkdir -p /usr/local/bin \ && curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ && chmod +x /usr/local/bin/kubectl \ + && curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-linux-x86_64.tar.gz \ + && tar xf google-cloud-cli-linux-x86_64.tar.gz \ + && mv google-cloud-sdk /usr/lib/google-cloud-sdk \ + && /usr/lib/google-cloud-sdk/install.sh --additional-components gke-gcloud-auth-plugin --path-update false --quiet \ + && ln -s /usr/lib/google-cloud-sdk/bin/gcloud /usr/bin/gcloud \ + && rm google-cloud-cli-linux-x86_64.tar.gz \ && curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 \ && chmod +x get_helm.sh \ && ./get_helm.sh \ @@ -270,10 +276,22 @@ export-end-to-end: FROM +end-to-end SAVE IMAGE aztecprotocol/end-to-end:$EARTHLY_GIT_HASH +export-end-to-end-arch: + FROM +end-to-end + ARG DIST_TAG="latest" + ARG ARCH + SAVE IMAGE --push aztecprotocol/end-to-end:${DIST_TAG}${ARCH:+-$ARCH} + export-e2e-test-images: BUILD +export-aztec BUILD +export-end-to-end +export-images-arch: + ARG DIST_TAG="latest" + ARG ARCH + BUILD +export-aztec-arch + BUILD +export-end-to-end-arch + format-check: FROM +build RUN yarn formatting diff --git a/yarn-project/end-to-end/scripts/network_test.sh b/yarn-project/end-to-end/scripts/network_test.sh index 2266d5e0d40..e5de5ca7185 100755 --- a/yarn-project/end-to-end/scripts/network_test.sh +++ b/yarn-project/end-to-end/scripts/network_test.sh @@ -142,7 +142,7 @@ kubectl wait pod -l app==pxe --for=condition=Ready -n "$NAMESPACE" --timeout=10m # Find 3 free ports between 9000 and 10000 FREE_PORTS=$(comm -23 <(seq 9000 10000 | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 3) -# Extract the two free ports from the list +# Extract the free ports from the list PXE_PORT=$(echo $FREE_PORTS | awk '{print $1}') ANVIL_PORT=$(echo $FREE_PORTS | awk '{print $2}') METRICS_PORT=$(echo $FREE_PORTS | awk '{print $3}') @@ -165,7 +165,7 @@ fi docker run --rm --network=host \ -v ~/.kube:/root/.kube \ - -e K8S=true \ + -e K8S=local \ -e INSTANCE_NAME="spartan" \ -e SPARTAN_DIR="/usr/src/spartan" \ -e NAMESPACE="$NAMESPACE" \ diff --git a/yarn-project/end-to-end/src/spartan/4epochs.test.ts b/yarn-project/end-to-end/src/spartan/4epochs.test.ts index feef5c9f243..35a16b1f896 100644 --- a/yarn-project/end-to-end/src/spartan/4epochs.test.ts +++ b/yarn-project/end-to-end/src/spartan/4epochs.test.ts @@ -7,9 +7,9 @@ import { jest } from '@jest/globals'; import { RollupCheatCodes } from '../../../aztec.js/src/utils/cheat_codes.js'; import { type TestWallets, setupTestWalletsWithTokens } from './setup_test_wallets.js'; -import { getConfig, isK8sConfig, startPortForward } from './utils.js'; +import { isK8sConfig, setupEnvironment, startPortForward } from './utils.js'; -const config = getConfig(process.env); +const config = setupEnvironment(process.env); describe('token transfer test', () => { jest.setTimeout(10 * 60 * 4000); // 40 minutes diff --git a/yarn-project/end-to-end/src/spartan/gating-passive.test.ts b/yarn-project/end-to-end/src/spartan/gating-passive.test.ts index 95921c66db9..6369f912a7a 100644 --- a/yarn-project/end-to-end/src/spartan/gating-passive.test.ts +++ b/yarn-project/end-to-end/src/spartan/gating-passive.test.ts @@ -11,10 +11,10 @@ import { applyValidatorKill, awaitL2BlockNumber, enableValidatorDynamicBootNode, - getConfig, isK8sConfig, restartBot, runAlertCheck, + setupEnvironment, startPortForward, } from './utils.js'; @@ -28,7 +28,7 @@ const qosAlerts: AlertConfig[] = [ }, ]; -const config = getConfig(process.env); +const config = setupEnvironment(process.env); if (!isK8sConfig(config)) { throw new Error('This test must be run in a k8s environment'); } diff --git a/yarn-project/end-to-end/src/spartan/proving.test.ts b/yarn-project/end-to-end/src/spartan/proving.test.ts index 8681f17601c..c4ea1fc0288 100644 --- a/yarn-project/end-to-end/src/spartan/proving.test.ts +++ b/yarn-project/end-to-end/src/spartan/proving.test.ts @@ -4,11 +4,11 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { jest } from '@jest/globals'; import { type ChildProcess } from 'child_process'; -import { getConfig, isK8sConfig, startPortForward } from './utils.js'; +import { isK8sConfig, setupEnvironment, startPortForward } from './utils.js'; jest.setTimeout(2_400_000); // 40 minutes -const config = getConfig(process.env); +const config = setupEnvironment(process.env); const debugLogger = createDebugLogger('aztec:spartan-test:proving'); const SLEEP_MS = 1000; diff --git a/yarn-project/end-to-end/src/spartan/reorg.test.ts b/yarn-project/end-to-end/src/spartan/reorg.test.ts index c315fe05def..92f724c77ea 100644 --- a/yarn-project/end-to-end/src/spartan/reorg.test.ts +++ b/yarn-project/end-to-end/src/spartan/reorg.test.ts @@ -8,13 +8,13 @@ import { type TestWallets, performTransfers, setupTestWalletsWithTokens } from ' import { applyProverFailure, deleteResourceByLabel, - getConfig, isK8sConfig, + setupEnvironment, startPortForward, waitForResourceByLabel, } from './utils.js'; -const config = getConfig(process.env); +const config = setupEnvironment(process.env); if (!isK8sConfig(config)) { throw new Error('This test must be run in a k8s environment'); } diff --git a/yarn-project/end-to-end/src/spartan/smoke.test.ts b/yarn-project/end-to-end/src/spartan/smoke.test.ts index 49bf8324a28..f58a2d6a469 100644 --- a/yarn-project/end-to-end/src/spartan/smoke.test.ts +++ b/yarn-project/end-to-end/src/spartan/smoke.test.ts @@ -6,9 +6,9 @@ import { createPublicClient, getAddress, getContract, http } from 'viem'; import { foundry } from 'viem/chains'; import { type AlertConfig } from '../quality_of_service/alert_checker.js'; -import { getConfig, isK8sConfig, runAlertCheck, startPortForward } from './utils.js'; +import { isK8sConfig, runAlertCheck, setupEnvironment, startPortForward } from './utils.js'; -const config = getConfig(process.env); +const config = setupEnvironment(process.env); const debugLogger = createDebugLogger('aztec:spartan-test:smoke'); diff --git a/yarn-project/end-to-end/src/spartan/transfer.test.ts b/yarn-project/end-to-end/src/spartan/transfer.test.ts index a1a9d7aea9a..79cd761cfd4 100644 --- a/yarn-project/end-to-end/src/spartan/transfer.test.ts +++ b/yarn-project/end-to-end/src/spartan/transfer.test.ts @@ -5,9 +5,9 @@ import { TokenContract } from '@aztec/noir-contracts.js'; import { jest } from '@jest/globals'; import { type TestWallets, setupTestWalletsWithTokens } from './setup_test_wallets.js'; -import { getConfig, isK8sConfig, startPortForward } from './utils.js'; +import { isK8sConfig, setupEnvironment, startPortForward } from './utils.js'; -const config = getConfig(process.env); +const config = setupEnvironment(process.env); describe('token transfer test', () => { jest.setTimeout(10 * 60 * 2000); // 20 minutes diff --git a/yarn-project/end-to-end/src/spartan/utils.ts b/yarn-project/end-to-end/src/spartan/utils.ts index 3152de9c53d..120b3b3adcd 100644 --- a/yarn-project/end-to-end/src/spartan/utils.ts +++ b/yarn-project/end-to-end/src/spartan/utils.ts @@ -1,7 +1,7 @@ import { createDebugLogger, sleep } from '@aztec/aztec.js'; import type { Logger } from '@aztec/foundation/log'; -import { exec, spawn } from 'child_process'; +import { exec, execSync, spawn } from 'child_process'; import path from 'path'; import { promisify } from 'util'; import { z } from 'zod'; @@ -13,7 +13,7 @@ const execAsync = promisify(exec); const logger = createDebugLogger('k8s-utils'); -const k8sConfigSchema = z.object({ +const k8sLocalConfigSchema = z.object({ INSTANCE_NAME: z.string().min(1, 'INSTANCE_NAME env variable must be set'), NAMESPACE: z.string().min(1, 'NAMESPACE env variable must be set'), HOST_PXE_PORT: z.coerce.number().min(1, 'HOST_PXE_PORT env variable must be set'), @@ -25,7 +25,13 @@ const k8sConfigSchema = z.object({ GRAFANA_PASSWORD: z.string().min(1, 'GRAFANA_PASSWORD env variable must be set'), METRICS_API_PATH: z.string().default('/api/datasources/proxy/uid/spartan-metrics-prometheus/api/v1/query'), SPARTAN_DIR: z.string().min(1, 'SPARTAN_DIR env variable must be set'), - K8S: z.literal('true'), + K8S: z.literal('local'), +}); + +const k8sGCloudConfigSchema = k8sLocalConfigSchema.extend({ + K8S: z.literal('gcloud'), + CLUSTER_NAME: z.string().min(1, 'CLUSTER_NAME env variable must be set'), + REGION: z.string().min(1, 'REGION env variable must be set'), }); const directConfigSchema = z.object({ @@ -34,18 +40,28 @@ const directConfigSchema = z.object({ K8S: z.literal('false'), }); -const envSchema = z.discriminatedUnion('K8S', [k8sConfigSchema, directConfigSchema]); +const envSchema = z.discriminatedUnion('K8S', [k8sLocalConfigSchema, k8sGCloudConfigSchema, directConfigSchema]); -export type K8sConfig = z.infer; +export type K8sLocalConfig = z.infer; +export type K8sGCloudConfig = z.infer; export type DirectConfig = z.infer; export type EnvConfig = z.infer; -export function getConfig(env: unknown): EnvConfig { - return envSchema.parse(env); +export function isK8sConfig(config: EnvConfig): config is K8sLocalConfig | K8sGCloudConfig { + return config.K8S === 'local' || config.K8S === 'gcloud'; } -export function isK8sConfig(config: EnvConfig): config is K8sConfig { - return config.K8S === 'true'; +export function isGCloudConfig(config: EnvConfig): config is K8sGCloudConfig { + return config.K8S === 'gcloud'; +} + +export function setupEnvironment(env: unknown): EnvConfig { + const config = envSchema.parse(env); + if (isGCloudConfig(config)) { + const command = `gcloud container clusters get-credentials ${config.CLUSTER_NAME} --region=${config.REGION}`; + execSync(command); + } + return config; } export async function startPortForward({