diff --git a/spartan/aztec-network/files/config/config-prover-env.sh b/spartan/aztec-network/files/config/config-prover-env.sh index 4ee7106cb73..11c4ad5aef2 100644 --- a/spartan/aztec-network/files/config/config-prover-env.sh +++ b/spartan/aztec-network/files/config/config-prover-env.sh @@ -1,11 +1,9 @@ -#!/bin/sh +#!/bin/bash set -eu -alias aztec='node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js' - # Pass the bootnode url as an argument # Ask the bootnode for l1 contract addresses -output=$(aztec get-node-info -u $1) +output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js get-node-info -u $1) echo "$output" @@ -22,7 +20,7 @@ governance_proposer_address=$(echo "$output" | grep -oP 'GovernanceProposer Addr governance_address=$(echo "$output" | grep -oP 'Governance Address: \K0x[a-fA-F0-9]{40}') # Write the addresses to a file in the shared volume -cat < /shared/contracts.env +cat < /shared/contracts/contracts.env export BOOTSTRAP_NODES=$boot_node_enr export ROLLUP_CONTRACT_ADDRESS=$rollup_address export REGISTRY_CONTRACT_ADDRESS=$registry_address @@ -36,4 +34,4 @@ export GOVERNANCE_PROPOSER_CONTRACT_ADDRESS=$governance_proposer_address export GOVERNANCE_CONTRACT_ADDRESS=$governance_address EOF -cat /shared/contracts.env +cat /shared/contracts/contracts.env diff --git a/spartan/aztec-network/files/config/config-validator-env.sh b/spartan/aztec-network/files/config/config-validator-env.sh index 174482492c4..71d03fbbc98 100644 --- a/spartan/aztec-network/files/config/config-validator-env.sh +++ b/spartan/aztec-network/files/config/config-validator-env.sh @@ -1,11 +1,10 @@ -#!/bin/sh +#!/bin/bash set -eu -alias aztec='node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js' # Pass the bootnode url as an argument # Ask the bootnode for l1 contract addresses -output=$(aztec get-node-info -u $1) +output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js get-node-info -u $1) echo "$output" @@ -28,7 +27,7 @@ private_key=$(jq -r ".[$INDEX]" /app/config/keys.json) # Write the addresses to a file in the shared volume -cat < /shared/contracts.env +cat < /shared/contracts/contracts.env export BOOTSTRAP_NODES=$boot_node_enr export ROLLUP_CONTRACT_ADDRESS=$rollup_address export REGISTRY_CONTRACT_ADDRESS=$registry_address @@ -45,4 +44,4 @@ export L1_PRIVATE_KEY=$private_key export SEQ_PUBLISHER_PRIVATE_KEY=$private_key EOF -cat /shared/contracts.env \ No newline at end of file +cat /shared/contracts/contracts.env diff --git a/spartan/aztec-network/files/config/deploy-l1-contracts.sh b/spartan/aztec-network/files/config/deploy-l1-contracts.sh index 66cc107f251..4d976821f04 100644 --- a/spartan/aztec-network/files/config/deploy-l1-contracts.sh +++ b/spartan/aztec-network/files/config/deploy-l1-contracts.sh @@ -1,9 +1,8 @@ -#!/bin/sh +#!/bin/bash set -exu CHAIN_ID=$1 -alias aztec='node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js' # Use default account, it is funded on our dev machine export PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" @@ -12,9 +11,9 @@ export PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4 output="" # if INIT_VALIDATORS is true, then we need to pass the validators flag to the deploy-l1-contracts command if [ "$INIT_VALIDATORS" = "true" ]; then - output=$(aztec deploy-l1-contracts --validators $2 --l1-chain-id $CHAIN_ID) + output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --validators $2 --l1-chain-id $CHAIN_ID) else - output=$(aztec deploy-l1-contracts --l1-chain-id $CHAIN_ID) + output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --l1-chain-id $CHAIN_ID) fi echo "$output" @@ -32,7 +31,7 @@ governance_proposer_address=$(echo "$output" | grep -oP 'GovernanceProposer Addr governance_address=$(echo "$output" | grep -oP 'Governance Address: \K0x[a-fA-F0-9]{40}') # Write the addresses to a file in the shared volume -cat < /shared/contracts.env +cat < /shared/contracts/contracts.env export ROLLUP_CONTRACT_ADDRESS=$rollup_address export REGISTRY_CONTRACT_ADDRESS=$registry_address export INBOX_CONTRACT_ADDRESS=$inbox_address @@ -45,4 +44,4 @@ export GOVERNANCE_PROPOSER_CONTRACT_ADDRESS=$governance_proposer_address export GOVERNANCE_CONTRACT_ADDRESS=$governance_address EOF -cat /shared/contracts.env +cat /shared/contracts/contracts.env diff --git a/spartan/aztec-network/files/config/setup-p2p-addresses.sh b/spartan/aztec-network/files/config/setup-p2p-addresses.sh new file mode 100644 index 00000000000..f4b2afce6f2 --- /dev/null +++ b/spartan/aztec-network/files/config/setup-p2p-addresses.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +POD_NAME=$(echo $HOSTNAME) + +if [ "${NETWORK_PUBLIC}" = "true" ]; then + # First try treating HOSTNAME as a pod name + NODE_NAME=$(kubectl get pod $POD_NAME -n ${NAMESPACE} -o jsonpath='{.spec.nodeName}' 2>/dev/null) + + # If that fails, HOSTNAME might be the node name itself + if [ $? -ne 0 ]; then + echo "Could not find pod $POD_NAME, assuming $POD_NAME is the node name" + NODE_NAME=$POD_NAME + fi + + EXTERNAL_IP=$(kubectl get node $NODE_NAME -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}') + + if [ -z "$EXTERNAL_IP" ]; then + echo "Warning: Could not find ExternalIP, falling back to InternalIP" + EXTERNAL_IP=$(kubectl get node $NODE_NAME -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') + fi + + TCP_ADDR="${EXTERNAL_IP}:${P2P_TCP_PORT}" + UDP_ADDR="${EXTERNAL_IP}:${P2P_UDP_PORT}" + +else + # Get pod IP for non-public networks + POD_IP=$(hostname -i) + TCP_ADDR="${POD_IP}:${P2P_TCP_PORT}" + UDP_ADDR="${POD_IP}:${P2P_UDP_PORT}" +fi + +# Write addresses to file for sourcing +echo "export P2P_TCP_ANNOUNCE_ADDR=${TCP_ADDR}" > /shared/p2p/p2p-addresses +echo "export P2P_TCP_LISTEN_ADDR=0.0.0.0:${P2P_TCP_PORT}" >> /shared/p2p/p2p-addresses +echo "export P2P_UDP_ANNOUNCE_ADDR=${UDP_ADDR}" >> /shared/p2p/p2p-addresses +echo "export P2P_UDP_LISTEN_ADDR=0.0.0.0:${P2P_UDP_PORT}" >> /shared/p2p/p2p-addresses + +echo "P2P addresses configured:" +cat /shared/p2p/p2p-addresses \ No newline at end of file diff --git a/spartan/aztec-network/files/config/setup-service-addresses.sh b/spartan/aztec-network/files/config/setup-service-addresses.sh new file mode 100644 index 00000000000..4594b7a7740 --- /dev/null +++ b/spartan/aztec-network/files/config/setup-service-addresses.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +set -ex + +# Function to get pod and node details +get_service_address() { + local SERVICE_LABEL=$1 + local PORT=$2 + local MAX_RETRIES=30 + local RETRY_INTERVAL=2 + local attempt=1 + + # Get pod name + while [ $attempt -le $MAX_RETRIES ]; do + POD_NAME=$(kubectl get pods -n ${NAMESPACE} -l app=${SERVICE_LABEL} -o jsonpath='{.items[0].metadata.name}') + if [ -n "$POD_NAME" ]; then + break + fi + echo "Attempt $attempt: Waiting for ${SERVICE_LABEL} pod to be available..." >&2 + sleep $RETRY_INTERVAL + attempt=$((attempt + 1)) + done + + if [ -z "$POD_NAME" ]; then + echo "Error: Failed to get ${SERVICE_LABEL} pod name after $MAX_RETRIES attempts" >&2 + return 1 + fi + echo "Pod name: [${POD_NAME}]" >&2 + + # Get node name + attempt=1 + NODE_NAME="" + while [ $attempt -le $MAX_RETRIES ]; do + NODE_NAME=$(kubectl get pod ${POD_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.nodeName}') + if [ -n "$NODE_NAME" ]; then + break + fi + echo "Attempt $attempt: Waiting for node name to be available..." >&2 + sleep $RETRY_INTERVAL + attempt=$((attempt + 1)) + done + + if [ -z "$NODE_NAME" ]; then + echo "Error: Failed to get node name after $MAX_RETRIES attempts" >&2 + return 1 + fi + echo "Node name: ${NODE_NAME}" >&2 + + # Get the node's external IP + NODE_IP=$(kubectl get node ${NODE_NAME} -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}') + echo "Node IP: ${NODE_IP}" >&2 + echo "http://${NODE_IP}:${PORT}" +} + +# Configure Ethereum address +if [ "${ETHEREUM_EXTERNAL_HOST}" != "" ]; then + ETHEREUM_ADDR="${ETHEREUM_EXTERNAL_HOST}" +elif [ "${NETWORK_PUBLIC}" = "true" ]; then + ETHEREUM_ADDR=$(get_service_address "ethereum" "${ETHEREUM_PORT}") +else + ETHEREUM_ADDR="http://${SERVICE_NAME}-ethereum.${NAMESPACE}:${ETHEREUM_PORT}" +fi + +# Configure Boot Node address +if [ "${BOOT_NODE_EXTERNAL_HOST}" != "" ]; then + BOOT_NODE_ADDR="${BOOT_NODE_EXTERNAL_HOST}" +elif [ "${NETWORK_PUBLIC}" = "true" ]; then + BOOT_NODE_ADDR=$(get_service_address "boot-node" "${BOOT_NODE_PORT}") +else + BOOT_NODE_ADDR="http://${SERVICE_NAME}-boot-node.${NAMESPACE}:${BOOT_NODE_PORT}" +fi + +# Configure Prover Node address +if [ "${PROVER_NODE_EXTERNAL_HOST}" != "" ]; then + PROVER_NODE_ADDR="${PROVER_NODE_EXTERNAL_HOST}" +elif [ "${NETWORK_PUBLIC}" = "true" ]; then + PROVER_NODE_ADDR=$(get_service_address "prover-node" "${PROVER_NODE_PORT}") +else + PROVER_NODE_ADDR="http://${SERVICE_NAME}-prover-node.${NAMESPACE}:${PROVER_NODE_PORT}" +fi + + +# Write addresses to file for sourcing +echo "export ETHEREUM_HOST=${ETHEREUM_ADDR}" >> /shared/config/service-addresses +echo "export BOOT_NODE_HOST=${BOOT_NODE_ADDR}" >> /shared/config/service-addresses +echo "export PROVER_NODE_HOST=${PROVER_NODE_ADDR}" >> /shared/config/service-addresses +echo "Addresses configured:" +cat /shared/config/service-addresses diff --git a/spartan/aztec-network/templates/_helpers.tpl b/spartan/aztec-network/templates/_helpers.tpl index 33f8dda0671..8afb0c4636d 100644 --- a/spartan/aztec-network/templates/_helpers.tpl +++ b/spartan/aztec-network/templates/_helpers.tpl @@ -50,37 +50,19 @@ app.kubernetes.io/name: {{ include "aztec-network.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} -{{- define "aztec-network.ethereumHost" -}} -{{- if .Values.ethereum.externalHost -}} -http://{{ .Values.ethereum.externalHost }}:{{ .Values.ethereum.service.port }} -{{- else -}} -http://{{ include "aztec-network.fullname" . }}-ethereum.{{ .Release.Namespace }}:{{ .Values.ethereum.service.port }} -{{- end -}} -{{- end -}} + {{- define "aztec-network.pxeUrl" -}} -{{- if .Values.pxe.externalHost -}} -http://{{ .Values.pxe.externalHost }}:{{ .Values.pxe.service.port }} -{{- else -}} -http://{{ include "aztec-network.fullname" . }}-pxe.{{ .Release.Namespace }}:{{ .Values.pxe.service.port }} -{{- end -}} +http://{{ include "aztec-network.fullname" . }}-pxe.{{ .Release.Namespace }}:{{ .Values.pxe.service.nodePort }} {{- end -}} {{- define "aztec-network.bootNodeUrl" -}} -{{- if .Values.bootNode.externalTcpHost -}} -http://{{ .Values.bootNode.externalTcpHost }}:{{ .Values.bootNode.service.nodePort }} -{{- else -}} http://{{ include "aztec-network.fullname" . }}-boot-node-0.{{ include "aztec-network.fullname" . }}-boot-node.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.bootNode.service.nodePort }} {{- end -}} -{{- end -}} {{- define "aztec-network.validatorUrl" -}} -{{- if .Values.validator.externalTcpHost -}} -http://{{ .Values.validator.externalTcpHost }}:{{ .Values.validator.service.nodePort }} -{{- else -}} http://{{ include "aztec-network.fullname" . }}-validator.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.validator.service.nodePort }} {{- end -}} -{{- end -}} {{- define "aztec-network.metricsHost" -}} http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }} @@ -123,3 +105,89 @@ http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }} {{- end -}} {{- end -}} {{- end -}} + +{{/* +P2P Setup Container +*/}} +{{- define "aztec-network.p2pSetupContainer" -}} +- name: setup-p2p-addresses + image: bitnami/kubectl + command: + - /bin/sh + - -c + - | + cp /scripts/setup-p2p-addresses.sh /tmp/setup-p2p-addresses.sh && \ + chmod +x /tmp/setup-p2p-addresses.sh && \ + /tmp/setup-p2p-addresses.sh + env: + - name: NETWORK_PUBLIC + value: "{{ .Values.network.public }}" + - name: NAMESPACE + value: {{ .Release.Namespace }} + - name: P2P_TCP_PORT + value: "{{ .Values.validator.service.p2pTcpPort }}" + - name: P2P_UDP_PORT + value: "{{ .Values.validator.service.p2pUdpPort }}" + volumeMounts: + - name: scripts + mountPath: /scripts + - name: p2p-addresses + mountPath: /shared/p2p +{{- end -}} + +{{/* +Service Address Setup Container +*/}} +{{- define "aztec-network.serviceAddressSetupContainer" -}} +- name: setup-service-addresses + image: bitnami/kubectl + command: + - /bin/bash + - -c + - | + cp /scripts/setup-service-addresses.sh /tmp/setup-service-addresses.sh && \ + chmod +x /tmp/setup-service-addresses.sh && \ + /tmp/setup-service-addresses.sh + env: + - name: NETWORK_PUBLIC + value: "{{ .Values.network.public }}" + - name: NAMESPACE + value: {{ .Release.Namespace }} + - name: EXTERNAL_ETHEREUM_HOST + value: "{{ .Values.ethereum.externalHost }}" + - name: ETHEREUM_PORT + value: "{{ .Values.ethereum.service.port }}" + - name: EXTERNAL_BOOT_NODE_HOST + value: "{{ .Values.bootNode.externalHost }}" + - name: BOOT_NODE_PORT + value: "{{ .Values.bootNode.service.nodePort }}" + - name: EXTERNAL_PROVER_NODE_HOST + value: "{{ .Values.proverNode.externalHost }}" + - name: PROVER_NODE_PORT + value: "{{ .Values.proverNode.service.nodePort }}" + - name: SERVICE_NAME + value: {{ include "aztec-network.fullname" . }} + volumeMounts: + - name: scripts + mountPath: /scripts + - name: config + mountPath: /shared/config +{{- end -}} + +{{/** +Anti-affinity when running in public network mode +*/}} +{{- define "aztec-network.publicAntiAffinity" -}} +affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - validator + - boot-node + - prover + topologyKey: "kubernetes.io/hostname" +{{- end -}} diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml index 5f29df22010..0643646c8a0 100644 --- a/spartan/aztec-network/templates/boot-node.yaml +++ b/spartan/aztec-network/templates/boot-node.yaml @@ -17,16 +17,25 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: boot-node spec: + {{- if .Values.network.public }} + hostNetwork: true + {{- include "aztec-network.publicAntiAffinity" . | nindent 6 }} + {{- end }} + serviceAccountName: {{ include "aztec-network.fullname" . }}-node initContainers: + {{- include "aztec-network.p2pSetupContainer" . | nindent 8 }} + {{- include "aztec-network.serviceAddressSetupContainer" . | nindent 8 }} - name: wait-for-ethereum - image: {{ .Values.images.curl.image }} + image: {{ .Values.images.aztec.image }} command: - - /bin/sh + - /bin/bash - -c - | + source /shared/config/service-addresses + echo "Awaiting ethereum node at ${ETHEREUM_HOST}" until curl -s -X POST -H 'Content-Type: application/json' \ -d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \ - {{ include "aztec-network.ethereumHost" . }} | grep -q reth; do + ${ETHEREUM_HOST} | grep -q reth; do echo "Waiting for Ethereum node..." sleep 5 done @@ -38,25 +47,31 @@ spec: done echo "OpenTelemetry collector is ready!" {{- end }} + volumeMounts: + - name: config + mountPath: /shared/config {{- if .Values.bootNode.deployContracts }} - - name: deploy-contracts + - name: deploy-l1-contracts image: {{ .Values.images.aztec.image }} command: [ - "/bin/sh", + "/bin/bash", "-c", - "cp /scripts/deploy-contracts.sh /tmp/deploy-contracts.sh && chmod +x /tmp/deploy-contracts.sh && /tmp/deploy-contracts.sh {{ .Values.ethereum.chainId }} \"{{ join "," .Values.validator.validatorAddresses }}\"" + "cp /scripts/deploy-l1-contracts.sh /tmp/deploy-l1-contracts.sh && \ + chmod +x /tmp/deploy-l1-contracts.sh && \ + source /shared/config/service-addresses && \ + /tmp/deploy-l1-contracts.sh {{ .Values.ethereum.chainId }} \"{{ join "," .Values.validator.validatorAddresses }}\"" ] volumeMounts: - name: scripts-output - mountPath: /shared + mountPath: /shared/contracts + - name: config + mountPath: /shared/config - name: scripts mountPath: /scripts env: - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - name: INIT_VALIDATORS - value: {{ not .Values.validator.external | quote }} + value: "true" - name: ETHEREUM_SLOT_DURATION value: "{{ .Values.ethereum.blockTime }}" - name: AZTEC_SLOT_DURATION @@ -70,12 +85,15 @@ spec: - name: boot-node image: {{ .Values.images.aztec.image }} command: - # sleep to allow dns name to be resolvable - [ - "/bin/bash", - "-c", - "sleep 30 && source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer --pxe", - ] + - /bin/bash + - -c + - | + sleep 30 && \ + source /shared/contracts/contracts.env && \ + source /shared/p2p/p2p-addresses && \ + source /shared/config/service-addresses && \ + env && \ + node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer --pxe startupProbe: httpGet: path: /status @@ -91,20 +109,24 @@ spec: timeoutSeconds: 30 failureThreshold: 3 volumeMounts: - {{- if .Values.bootNode.deployContracts }} + - name: p2p-addresses + mountPath: /shared/p2p + - name: config + mountPath: /shared/config + {{- if .Values.bootNode.deployContracts }} - name: scripts-output - mountPath: /shared - {{- else }} + mountPath: /shared/contracts + {{- else }} - name: contracts-env - mountPath: /shared/contracts.env + mountPath: /shared/contracts/contracts.env subPath: contracts.env - {{- end }} + {{- end }} env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: PORT + - name: AZTEC_PORT value: "{{ .Values.bootNode.service.nodePort }}" - name: LOG_LEVEL value: "{{ .Values.bootNode.logLevel }}" @@ -112,8 +134,6 @@ spec: value: "1" - name: DEBUG value: "{{ .Values.bootNode.debug }}" - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - name: P2P_ENABLED value: "{{ .Values.bootNode.p2p.enabled }}" - name: COINBASE @@ -126,22 +146,6 @@ spec: value: "{{ .Values.bootNode.sequencer.maxSecondsBetweenBlocks }}" - name: SEQ_MIN_TX_PER_BLOCK value: "{{ .Values.bootNode.sequencer.minTxsPerBlock }}" - - name: P2P_TCP_ANNOUNCE_ADDR - {{- if .Values.bootNode.externalTcpHost }} - value: "{{ .Values.bootNode.externalTcpHost }}:{{ .Values.bootNode.service.p2pTcpPort }}" - {{- else }} - value: "$(POD_IP):{{ .Values.bootNode.service.p2pTcpPort }}" - {{- end }} - - name: P2P_UDP_ANNOUNCE_ADDR - {{- if .Values.bootNode.externalUdpHost }} - value: "{{ .Values.bootNode.externalUdpHost }}:{{ .Values.bootNode.service.p2pUdpPort }}" - {{- else }} - value: "$(POD_IP):{{ .Values.bootNode.service.p2pUdpPort }}" - {{- end }} - - name: P2P_TCP_LISTEN_ADDR - value: "0.0.0.0:{{ .Values.bootNode.service.p2pTcpPort }}" - - name: P2P_UDP_LISTEN_ADDR - value: "0.0.0.0:{{ .Values.bootNode.service.p2pUdpPort }}" - name: VALIDATOR_PRIVATE_KEY value: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" - name: OTEL_RESOURCE_ATTRIBUTES @@ -170,10 +174,14 @@ spec: resources: {{- toYaml .Values.bootNode.resources | nindent 12 }} volumes: + - name: p2p-addresses + emptyDir: {} + - name: config + emptyDir: {} {{- if .Values.bootNode.deployContracts }} - name: scripts configMap: - name: {{ include "aztec-network.fullname" . }}-deploy-contracts-script + name: {{ include "aztec-network.fullname" . }}-scripts - name: scripts-output emptyDir: {} {{- else }} @@ -181,18 +189,7 @@ spec: configMap: name: {{ include "aztec-network.fullname" . }}-contracts-env {{- end }} -{{- if .Values.bootNode.deployContracts }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "aztec-network.fullname" . }}-deploy-contracts-script - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -data: - deploy-contracts.sh: | - {{ .Files.Get "files/config/deploy-l1-contracts.sh" | nindent 4 }} -{{- else }} +{{- if not .Values.bootNode.deployContracts }} --- apiVersion: v1 kind: ConfigMap @@ -209,6 +206,7 @@ data: export FEE_JUICE_CONTRACT_ADDRESS={{ .Values.bootNode.contracts.feeJuiceAddress }} export FEE_JUICE_PORTAL_CONTRACT_ADDRESS={{ .Values.bootNode.contracts.feeJuicePortalAddress }} {{- end }} +{{if not .Values.network.public }} --- # Headless service for StatefulSet DNS entries apiVersion: v1 @@ -230,43 +228,4 @@ spec: protocol: UDP - port: {{ .Values.bootNode.service.nodePort }} name: node ---- -{{if .Values.network.public }} -apiVersion: v1 -kind: Service -metadata: - name: boot-node-lb-tcp - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -spec: - type: LoadBalancer - selector: - {{- include "aztec-network.selectorLabels" . | nindent 4 }} - app: boot-node - ports: - - port: {{ .Values.bootNode.service.p2pTcpPort }} - name: p2p-tpc - - port: {{ .Values.bootNode.service.nodePort }} - name: node ---- -apiVersion: v1 -kind: Service -metadata: - name: boot-node-lb-udp - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" - service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -spec: - type: LoadBalancer - selector: - {{- include "aztec-network.selectorLabels" . | nindent 4 }} - app: boot-node - ports: - - port: {{ .Values.bootNode.service.p2pUdpPort }} - name: p2p-udp - protocol: UDP ---- {{ end }} diff --git a/spartan/aztec-network/templates/deploy-l1-verifier.yaml b/spartan/aztec-network/templates/deploy-l1-verifier.yaml index 486db8d24ca..cab6f8a78ab 100644 --- a/spartan/aztec-network/templates/deploy-l1-verifier.yaml +++ b/spartan/aztec-network/templates/deploy-l1-verifier.yaml @@ -1,4 +1,4 @@ -{{- if .Values.network.setupL2Contracts }} +{{- if and .Values.network.setupL2Contracts .Values.jobs.deployL1Verifier.enable }} apiVersion: batch/v1 kind: Job metadata: @@ -13,6 +13,12 @@ spec: app: deploy-l1-verifier spec: restartPolicy: OnFailure + volumes: + - name: config + emptyDir: {} + - name: scripts + configMap: + name: {{ include "aztec-network.fullname" . }}-scripts containers: - name: deploy-l1-verifier image: {{ .Values.images.aztec.image }} @@ -21,38 +27,69 @@ spec: - -c - | set -e + # Install kubectl + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + mv kubectl /usr/local/bin/ - [ $ENABLE = "true" ] || exit 0 + # Set up kubeconfig using service account credentials + export KUBECONFIG=/tmp/kubeconfig + kubectl config set-cluster default --server=https://kubernetes.default.svc --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + kubectl config set-credentials default --token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + kubectl config set-context default --cluster=default --user=default + kubectl config use-context default - until curl -s -X GET "$AZTEC_NODE_URL/status"; do - echo "Waiting for Aztec node $AZTEC_NODE_URL..." + cp /scripts/setup-service-addresses.sh /tmp/setup-service-addresses.sh + chmod +x /tmp/setup-service-addresses.sh + /tmp/setup-service-addresses.sh + source /shared/config/service-addresses + + until curl -s -X GET "$BOOT_NODE_HOST/status"; do + echo "Waiting for Aztec node $BOOT_NODE_HOST..." sleep 5 done echo "Boot node is ready!" export ROLLUP_CONTRACT_ADDRESS=$(curl -X POST -H 'Content-Type: application/json' \ -d '{"jsonrpc":"2.0","method":"node_getL1ContractAddresses","params":[],"id":1}' \ - "$AZTEC_NODE_URL" \ + "$BOOT_NODE_HOST" \ | jq -r '.result.rollupAddress.value') echo "Rollup contract address: $ROLLUP_CONTRACT_ADDRESS" node /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-verifier --verifier real echo "L1 verifier deployed" env: - - name: ENABLE - value: {{ .Values.jobs.deployL1Verifier.enable | quote }} - name: NODE_NO_WARNINGS value: "1" - name: DEBUG value: "aztec:*" - name: LOG_LEVEL value: "debug" - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - name: L1_CHAIN_ID value: {{ .Values.ethereum.chainId | quote }} - name: PRIVATE_KEY value: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" - - name: AZTEC_NODE_URL - value: {{ include "aztec-network.bootNodeUrl" . | quote }} + - name: NETWORK_PUBLIC + value: "{{ .Values.network.public }}" + - name: NAMESPACE + value: {{ .Release.Namespace }} + - name: EXTERNAL_ETHEREUM_HOST + value: "{{ .Values.ethereum.externalHost }}" + - name: ETHEREUM_PORT + value: "{{ .Values.ethereum.service.port }}" + - name: EXTERNAL_BOOT_NODE_HOST + value: "{{ .Values.bootNode.externalHost }}" + - name: BOOT_NODE_PORT + value: "{{ .Values.bootNode.service.nodePort }}" + - name: EXTERNAL_PROVER_NODE_HOST + value: "{{ .Values.proverNode.externalHost }}" + - name: PROVER_NODE_PORT + value: "{{ .Values.proverNode.service.nodePort }}" + - name: SERVICE_NAME + value: {{ include "aztec-network.fullname" . }} + volumeMounts: + - name: config + mountPath: /shared/config + - name: scripts + mountPath: /scripts {{ end }} diff --git a/spartan/aztec-network/templates/prover-agent.yaml b/spartan/aztec-network/templates/prover-agent.yaml index f929daa4b79..04f58284d21 100644 --- a/spartan/aztec-network/templates/prover-agent.yaml +++ b/spartan/aztec-network/templates/prover-agent.yaml @@ -17,20 +17,32 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: prover-agent spec: + serviceAccountName: {{ include "aztec-network.fullname" . }}-node {{- if .Values.proverAgent.nodeSelector }} nodeSelector: {{- toYaml .Values.proverAgent.nodeSelector | nindent 8 }} {{- end }} + {{- if .Values.network.public }} + hostNetwork: true + {{- end }} + volumes: + - name: config + emptyDir: {} + - name: scripts + configMap: + name: {{ include "aztec-network.fullname" . }}-scripts initContainers: + {{- include "aztec-network.serviceAddressSetupContainer" . | nindent 8 }} - name: wait-for-prover-node - image: {{ .Values.images.curl.image }} + image: {{ .Values.images.aztec.image }} command: - - /bin/sh + - /bin/bash - -c - | - until curl -s -X POST "$PROVER_JOB_SOURCE_URL/status"; do - echo "Waiting for Prover node $PROVER_JOB_SOURCE_URL ..." + source /shared/config/service-addresses + until curl -s -X POST ${PROVER_NODE_HOST}/status; do + echo "Waiting for Prover node ${PROVER_NODE_HOST} ..." sleep 5 done echo "Prover node is ready!" @@ -41,18 +53,26 @@ spec: done echo "OpenTelemetry collector is ready!" {{- end }} - env: - - name: PROVER_JOB_SOURCE_URL - value: "http://{{ include "aztec-network.fullname" . }}-prover-node.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.proverNode.service.nodePort }}" + volumeMounts: + - name: config + mountPath: /shared/config containers: - name: prover-agent image: "{{ .Values.images.aztec.image }}" imagePullPolicy: {{ .Values.images.aztec.pullPolicy }} + volumeMounts: + - name: config + mountPath: /shared/config command: - "/bin/bash" - "-c" - - "node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --prover" + - | + source /shared/config/service-addresses && \ + PROVER_JOB_SOURCE_URL=${PROVER_NODE_HOST} \ + node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --prover env: + - name: AZTEC_PORT + value: "{{ .Values.proverAgent.service.nodePort }}" - name: LOG_LEVEL value: "{{ .Values.proverAgent.logLevel }}" - name: LOG_JSON @@ -61,8 +81,6 @@ spec: value: "{{ .Values.proverAgent.debug }}" - name: PROVER_REAL_PROOFS value: "{{ .Values.proverAgent.realProofs }}" - - name: PROVER_JOB_SOURCE_URL - value: "http://{{ include "aztec-network.fullname" . }}-prover-node.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.proverNode.service.nodePort }}" - name: PROVER_AGENT_ENABLED value: "true" - name: PROVER_AGENT_CONCURRENCY diff --git a/spartan/aztec-network/templates/prover-node.yaml b/spartan/aztec-network/templates/prover-node.yaml index ff11cbf1ee3..6b7506149a2 100644 --- a/spartan/aztec-network/templates/prover-node.yaml +++ b/spartan/aztec-network/templates/prover-node.yaml @@ -17,16 +17,24 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: prover-node spec: + {{- if .Values.network.public }} + hostNetwork: true + {{- include "aztec-network.publicAntiAffinity" . | nindent 6 }} + {{- end }} + serviceAccountName: {{ include "aztec-network.fullname" . }}-node initContainers: + {{- include "aztec-network.serviceAddressSetupContainer" . | nindent 8 }} + {{- include "aztec-network.p2pSetupContainer" . | nindent 8 }} - name: wait-for-services - image: {{ .Values.images.curl.image }} + image: {{ .Values.images.aztec.image }} command: - - /bin/sh + - /bin/bash - -c - | + source /shared/config/service-addresses until curl -s -X POST -H 'Content-Type: application/json' \ -d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \ - {{ include "aztec-network.ethereumHost" . }} | grep -q reth; do + ${ETHEREUM_HOST} | grep -q reth; do echo "Waiting for Ethereum node..." sleep 5 done @@ -38,26 +46,31 @@ spec: done echo "OpenTelemetry collector is ready!" {{- end }} - until curl --head --silent {{ include "aztec-network.bootNodeUrl" . }}/status; do + until curl --head --silent $BOOT_NODE_HOST/status; do echo "Waiting for boot node..." sleep 5 done echo "Boot node is ready!" + volumeMounts: + - name: config + mountPath: /shared/config - name: configure-prover-env image: "{{ .Values.images.aztec.image }}" imagePullPolicy: {{ .Values.images.aztec.pullPolicy }} command: - - "/bin/sh" + - "/bin/bash" - "-c" - - "cp /scripts/configure-prover-env.sh /tmp/configure-prover-env.sh && chmod +x /tmp/configure-prover-env.sh && /tmp/configure-prover-env.sh {{ include "aztec-network.bootNodeUrl" . }}" + - "cp /scripts/configure-prover-env.sh /tmp/configure-prover-env.sh && \ + chmod +x /tmp/configure-prover-env.sh && \ + source /shared/config/service-addresses && \ + /tmp/configure-prover-env.sh ${BOOT_NODE_HOST}" volumeMounts: - - name: shared-volume - mountPath: /shared + - name: contracts-env + mountPath: /shared/contracts - name: scripts mountPath: /scripts - env: - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} + - name: config + mountPath: /shared/config containers: - name: prover-node @@ -66,16 +79,25 @@ spec: command: - "/bin/bash" - "-c" - - "source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --prover-node --archiver" + - | + source /shared/contracts/contracts.env && \ + source /shared/p2p/p2p-addresses && \ + source /shared/config/service-addresses && \ + env && \ + node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --prover-node --archiver volumeMounts: - - name: shared-volume - mountPath: /shared + - name: contracts-env + mountPath: /shared/contracts + - name: p2p-addresses + mountPath: /shared/p2p + - name: config + mountPath: /shared/config env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: PORT + - name: AZTEC_PORT value: "{{ .Values.proverNode.service.nodePort }}" - name: LOG_LEVEL value: "{{ .Values.proverNode.logLevel }}" @@ -83,8 +105,6 @@ spec: value: "1" - name: DEBUG value: "{{ .Values.proverNode.debug }}" - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - name: PROVER_REAL_PROOFS value: "{{ .Values.proverNode.realProofs }}" - name: PROVER_AGENT_ENABLED @@ -106,18 +126,6 @@ spec: value: "{{ .Values.ethereum.chainId }}" - name: P2P_ENABLED value: "{{ .Values.proverNode.p2pEnabled }}" - - name: P2P_TCP_ANNOUNCE_ADDR - {{- if .Values.proverNode.externalTcpHost }} - value: "{{ .Values.proverNode.externalTcpHost }}:{{ .Values.proverNode.service.p2pTcpPort }}" - {{- else }} - value: "$(POD_IP):{{ .Values.proverNode.service.p2pTcpPort }}" - {{- end }} - - name: P2P_UDP_ANNOUNCE_ADDR - {{- if .Values.proverNode.externalUdpHost }} - value: "{{ .Values.proverNode.externalUdpHost }}:{{ .Values.proverNode.service.p2pUdpPort }}" - {{- else }} - value: "$(POD_IP):{{ .Values.proverNode.service.p2pUdpPort }}" - {{- end }} - name: P2P_TCP_LISTEN_ADDR value: "0.0.0.0:{{ .Values.proverNode.service.p2pTcpPort }}" - name: P2P_UDP_LISTEN_ADDR @@ -140,7 +148,13 @@ spec: volumes: - name: scripts configMap: - name: {{ include "aztec-network.fullname" . }}-configure-prover-env + name: {{ include "aztec-network.fullname" . }}-scripts + - name: contracts-env + emptyDir: {} + - name: p2p-addresses + emptyDir: {} + - name: config + emptyDir: {} volumeClaimTemplates: - metadata: name: shared-volume @@ -151,16 +165,7 @@ spec: resources: requests: storage: {{ .Values.proverNode.storage }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "aztec-network.fullname" . }}-configure-prover-env - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -data: - configure-prover-env.sh: | - {{ .Files.Get "files/config/config-prover-env.sh" | nindent 4 }} +{{if not .Values.network.public }} --- apiVersion: v1 kind: Service @@ -181,43 +186,4 @@ spec: - port: {{ .Values.proverNode.service.p2pUdpPort }} name: p2p-udp protocol: UDP ---- -{{if .Values.proverNode.public }} -apiVersion: v1 -kind: Service -metadata: - name: prover-node-lb-tcp - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -spec: - type: LoadBalancer - selector: - {{- include "aztec-network.selectorLabels" . | nindent 4 }} - app: prover-node - ports: - - port: {{ .Values.proverNode.service.nodePort }} - name: node - - port: {{ .Values.proverNode.service.p2pTcpPort }} - name: p2p-tcp ---- -apiVersion: v1 -kind: Service -metadata: - name: prover-node-lb-udp - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" - service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -spec: - type: LoadBalancer - selector: - {{- include "aztec-network.selectorLabels" . | nindent 4 }} - app: prover-node - ports: - - port: {{ .Values.proverNode.service.p2pUdpPort }} - name: p2p-udp - protocol: UDP ---- {{ end }} diff --git a/spartan/aztec-network/templates/pxe.yaml b/spartan/aztec-network/templates/pxe.yaml index d229227e2c8..94a8a87886c 100644 --- a/spartan/aztec-network/templates/pxe.yaml +++ b/spartan/aztec-network/templates/pxe.yaml @@ -16,17 +16,36 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: pxe spec: + serviceAccountName: {{ include "aztec-network.fullname" . }}-node + {{- if .Values.network.public }} + hostNetwork: true + {{- end }} + volumes: + - name: config + emptyDir: {} + - name: scripts + configMap: + name: {{ include "aztec-network.fullname" . }}-scripts + - name: scripts-output + emptyDir: {} initContainers: + {{- include "aztec-network.serviceAddressSetupContainer" . | nindent 8 }} - name: wait-for-boot-node image: {{ .Values.images.curl.image }} command: - /bin/sh - -c - | - until curl --head --silent {{ include "aztec-network.bootNodeUrl" . }}/status; do + source /shared/config/service-addresses + until curl --head --silent ${BOOT_NODE_HOST}/status; do echo "Waiting for boot node..." sleep 5 done + volumeMounts: + - name: config + mountPath: /shared/config + {{- if not .Values.network.public }} + # We only need to wait for the validator service if the network is not public - name: wait-for-validator-service image: {{ .Values.images.curl.image }} command: @@ -37,19 +56,30 @@ spec: echo "Waiting for validator service..." sleep 5 done + {{- end }} containers: - name: pxe image: "{{ .Values.images.aztec.image }}" + volumeMounts: + - name: config + mountPath: /shared/config command: - "/bin/bash" - "-c" - - > + - | + source /shared/config/service-addresses + {{- if .Values.network.public }} + # If the network is public, we need to use the boot node URL + export AZTEC_NODE_URL=${BOOT_NODE_HOST} + {{- else }} + # If the network is not public, we can use the validator URL + export AZTEC_NODE_URL={{ include "aztec-network.validatorUrl" . }} + {{- end }} + echo "AZTEC_NODE_URL=${AZTEC_NODE_URL}" node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --pxe env: - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - - name: AZTEC_NODE_URL - value: {{ include "aztec-network.validatorUrl" . | quote }} + - name: AZTEC_PORT + value: "{{ .Values.pxe.service.nodePort }}" - name: LOG_JSON value: "1" - name: LOG_LEVEL @@ -60,7 +90,7 @@ spec: value: "{{ .Values.pxe.proverEnabled }}" ports: - name: http - containerPort: {{ .Values.pxe.service.port }} + containerPort: {{ .Values.pxe.service.nodePort }} protocol: TCP readinessProbe: exec: @@ -70,7 +100,7 @@ spec: - | curl -s -X POST -H 'content-type: application/json' \ -d '{"jsonrpc":"2.0","method":"pxe_isGlobalStateSynchronized","params":[],"id":67}' \ - 127.0.0.1:{{ .Values.pxe.service.port }} | grep -q '"result":true' + 127.0.0.1:{{ .Values.pxe.service.nodePort }} | grep -q '"result":true' initialDelaySeconds: {{ .Values.pxe.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.pxe.readinessProbe.periodSeconds }} timeoutSeconds: {{ .Values.pxe.readinessProbe.timeoutSeconds }} @@ -92,8 +122,8 @@ spec: app: pxe ports: - protocol: TCP - port: {{ .Values.pxe.service.port }} - targetPort: {{ .Values.pxe.service.targetPort }} + port: {{ .Values.pxe.service.nodePort }} + targetPort: {{ .Values.pxe.service.nodePort }} {{- if and (eq .Values.pxe.service.type "NodePort") .Values.pxe.service.nodePort }} nodePort: {{ .Values.pxe.service.nodePort }} {{- end }} @@ -103,6 +133,10 @@ apiVersion: v1 kind: Service metadata: name: pxe-lb + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" + service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" labels: {{- include "aztec-network.labels" . | nindent 4 }} spec: @@ -112,8 +146,8 @@ spec: app: pxe ports: - protocol: TCP - port: {{ .Values.pxe.service.port }} - targetPort: {{ .Values.pxe.service.targetPort }} + port: {{ .Values.pxe.service.nodePort }} + targetPort: {{ .Values.pxe.service.nodePort }} {{- if and (eq .Values.pxe.service.type "NodePort") .Values.pxe.service.nodePort }} nodePort: {{ .Values.pxe.service.nodePort }} {{- end }} diff --git a/spartan/aztec-network/templates/rbac.yaml b/spartan/aztec-network/templates/rbac.yaml new file mode 100644 index 00000000000..a0e8e68cd11 --- /dev/null +++ b/spartan/aztec-network/templates/rbac.yaml @@ -0,0 +1,58 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "aztec-network.fullname" . }}-node + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "aztec-network.fullname" . }}-node + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +rules: +- apiGroups: [""] + resources: ["services", "pods"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "aztec-network.fullname" . }}-node + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "aztec-network.fullname" . }}-node +subjects: +- kind: ServiceAccount + name: {{ include "aztec-network.fullname" . }}-node +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "aztec-network.fullname" . }}-node + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "aztec-network.fullname" . }}-node + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "aztec-network.fullname" . }}-node +subjects: +- kind: ServiceAccount + name: {{ include "aztec-network.fullname" . }}-node + namespace: {{ .Release.Namespace }} diff --git a/spartan/aztec-network/templates/reth.yaml b/spartan/aztec-network/templates/reth.yaml index 7312bab7ad5..d6230ecf0ad 100644 --- a/spartan/aztec-network/templates/reth.yaml +++ b/spartan/aztec-network/templates/reth.yaml @@ -16,6 +16,9 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: ethereum spec: + {{- if .Values.network.public }} + hostNetwork: true + {{- end }} containers: - name: ethereum image: "{{ .Values.images.reth.image }}" @@ -39,21 +42,6 @@ spec: mountPath: /data - name: genesis mountPath: /genesis - # readinessProbe: - # exec: - # command: - # - sh - # - -c - # - | - # wget -qO- --post-data='{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \ - # --header='Content-Type: application/json' \ - # 127.0.0.1:{{ .Values.ethereum.service.port }} \ - # | grep -q 'reth' - # initialDelaySeconds: {{ .Values.ethereum.readinessProbe.initialDelaySeconds }} - # periodSeconds: {{ .Values.ethereum.readinessProbe.periodSeconds }} - # timeoutSeconds: {{ .Values.ethereum.readinessProbe.timeoutSeconds }} - # successThreshold: {{ .Values.ethereum.readinessProbe.successThreshold }} - # failureThreshold: {{ .Values.ethereum.readinessProbe.failureThreshold }} resources: {{- toYaml .Values.ethereum.resources | nindent 12 }} volumes: @@ -63,6 +51,7 @@ spec: - name: genesis configMap: name: {{ include "aztec-network.fullname" . }}-reth-genesis +{{if not .Values.network.public }} --- apiVersion: v1 kind: Service @@ -82,26 +71,6 @@ spec: {{- if and (eq .Values.ethereum.service.type "NodePort") .Values.ethereum.service.nodePort }} nodePort: {{ .Values.ethereum.service.nodePort }} {{- end }} ---- -{{if .Values.network.public }} -apiVersion: v1 -kind: Service -metadata: - name: ethereum-lb - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -spec: - type: LoadBalancer - selector: - {{- include "aztec-network.selectorLabels" . | nindent 4 }} - app: ethereum - ports: - - protocol: TCP - port: {{ .Values.ethereum.service.port }} - targetPort: {{ .Values.ethereum.service.targetPort }} - {{- if and (eq .Values.ethereum.service.type "NodePort") .Values.ethereum.service.nodePort }} - nodePort: {{ .Values.ethereum.service.nodePort }} - {{- end }} {{ end }} --- apiVersion: v1 diff --git a/spartan/aztec-network/templates/scripts-configmap.yaml b/spartan/aztec-network/templates/scripts-configmap.yaml new file mode 100644 index 00000000000..bc86aabbd36 --- /dev/null +++ b/spartan/aztec-network/templates/scripts-configmap.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "aztec-network.fullname" . }}-scripts + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +data: + setup-service-addresses.sh: | + {{ .Files.Get "files/config/setup-service-addresses.sh" | nindent 4 }} + setup-p2p-addresses.sh: | + {{ .Files.Get "files/config/setup-p2p-addresses.sh" | nindent 4 }} + configure-validator-env.sh: | + {{ .Files.Get "files/config/config-validator-env.sh" | nindent 4 }} + configure-prover-env.sh: | + {{ .Files.Get "files/config/config-prover-env.sh" | nindent 4 }} + deploy-l1-contracts.sh: | + {{ .Files.Get "files/config/deploy-l1-contracts.sh" | nindent 4 }} diff --git a/spartan/aztec-network/templates/setup-l2-contracts.yaml b/spartan/aztec-network/templates/setup-l2-contracts.yaml index df05ffd20cc..56cf8fc57f2 100644 --- a/spartan/aztec-network/templates/setup-l2-contracts.yaml +++ b/spartan/aztec-network/templates/setup-l2-contracts.yaml @@ -13,16 +13,46 @@ spec: app: setup-l2-contracts spec: restartPolicy: OnFailure + serviceAccountName: {{ include "aztec-network.fullname" . }}-node + volumes: + - name: scripts + configMap: + name: {{ include "aztec-network.fullname" . }}-scripts + - name: config + emptyDir: {} containers: - name: setup-l2-contracts image: {{ .Values.images.aztec.image }} + volumeMounts: + - name: config + mountPath: /shared/config + - name: scripts + mountPath: /scripts command: - /bin/bash - -c - | + # Install kubectl + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + mv kubectl /usr/local/bin/ + + # Set up kubeconfig using service account credentials + export KUBECONFIG=/tmp/kubeconfig + kubectl config set-cluster default --server=https://kubernetes.default.svc --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + kubectl config set-credentials default --token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + kubectl config set-context default --cluster=default --user=default + kubectl config use-context default + + cp /scripts/setup-service-addresses.sh /tmp/setup-service-addresses.sh + chmod +x /tmp/setup-service-addresses.sh + /tmp/setup-service-addresses.sh + source /shared/config/service-addresses + export AZTEC_NODE_URL=$BOOT_NODE_HOST + export PXE_URL=$BOOT_NODE_HOST until curl -s -X POST -H 'content-type: application/json' \ -d '{"jsonrpc":"2.0","method":"pxe_getNodeInfo","params":[],"id":67}' \ - {{ include "aztec-network.pxeUrl" . }} | grep -q '"enr:-'; do + $PXE_URL | grep -q '"enr:-'; do echo "Waiting for PXE service..." sleep 5 done @@ -31,10 +61,26 @@ spec: node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js setup-protocol-contracts --skipProofWait --l1-chain-id {{ .Values.ethereum.chainId }} echo "L2 contracts initialized" env: - - name: PXE_URL - value: {{ include "aztec-network.bootNodeUrl" . | quote }} - name: DEBUG value: "aztec:*" - name: LOG_LEVEL value: "debug" + - name: NETWORK_PUBLIC + value: "{{ .Values.network.public }}" + - name: NAMESPACE + value: {{ .Release.Namespace }} + - name: EXTERNAL_ETHEREUM_HOST + value: "{{ .Values.ethereum.externalHost }}" + - name: ETHEREUM_PORT + value: "{{ .Values.ethereum.service.port }}" + - name: EXTERNAL_BOOT_NODE_HOST + value: "{{ .Values.bootNode.externalHost }}" + - name: BOOT_NODE_PORT + value: "{{ .Values.bootNode.service.nodePort }}" + - name: EXTERNAL_PROVER_NODE_HOST + value: "{{ .Values.proverNode.externalHost }}" + - name: PROVER_NODE_PORT + value: "{{ .Values.proverNode.service.nodePort }}" + - name: SERVICE_NAME + value: {{ include "aztec-network.fullname" . }} {{ end }} diff --git a/spartan/aztec-network/templates/transaction-bot.yaml b/spartan/aztec-network/templates/transaction-bot.yaml index 9f1239fcc2b..cd5b88a13bd 100644 --- a/spartan/aztec-network/templates/transaction-bot.yaml +++ b/spartan/aztec-network/templates/transaction-bot.yaml @@ -17,38 +17,64 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: bot spec: + {{- if .Values.network.public }} + hostNetwork: true + {{- end }} + serviceAccountName: {{ include "aztec-network.fullname" . }}-node + volumes: + - name: config + emptyDir: {} + - name: scripts + configMap: + name: {{ include "aztec-network.fullname" . }}-scripts + - name: scripts-output + emptyDir: {} initContainers: + {{- include "aztec-network.serviceAddressSetupContainer" . | nindent 8 }} - name: wait-for-aztec-node image: "{{ .Values.images.curl.image }}" - env: - - name: AZTEC_NODE_URL - {{- if .Values.bot.nodeUrl }} - value: "{{ .Values.bot.nodeUrl }}" - {{- else }} - value: {{ include "aztec-network.validatorUrl" . | quote }} - {{- end }} command: - /bin/sh - -c - | - until curl -s $(AZTEC_NODE_URL)/status; do echo waiting for aztec-node; sleep 2; done + source /shared/config/service-addresses + {{- if .Values.bot.nodeUrl }} + export AZTEC_NODE_URL={{ .Values.bot.nodeUrl }} + {{- else if .Values.network.public }} + export AZTEC_NODE_URL=${BOOT_NODE_HOST} + {{- else }} + export AZTEC_NODE_URL={{ include "aztec-network.validatorUrl" . }} + {{- end }} + echo "AZTEC_NODE_URL=${AZTEC_NODE_URL}" + until curl -s ${AZTEC_NODE_URL}/status; do echo waiting for aztec-node; sleep 2; done + volumeMounts: + - name: config + mountPath: /shared/config containers: - name: transaction-bot image: "{{ .Values.images.aztec.image }}" + volumeMounts: + - name: config + mountPath: /shared/config + - name: scripts + mountPath: /scripts command: - "/bin/bash" - "-c" - - > - node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --pxe --bot - env: - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - - name: AZTEC_NODE_URL + - | + source /shared/config/service-addresses {{- if .Values.bot.nodeUrl }} - value: "{{ .Values.bot.nodeUrl }}" + export AZTEC_NODE_URL={{ .Values.bot.nodeUrl }} + {{- else if .Values.network.public }} + export AZTEC_NODE_URL=${BOOT_NODE_HOST} {{- else }} - value: {{ include "aztec-network.validatorUrl" . | quote }} + export AZTEC_NODE_URL={{ include "aztec-network.validatorUrl" . }} {{- end }} + echo "AZTEC_NODE_URL=${AZTEC_NODE_URL}" + node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --pxe --bot + env: + - name: AZTEC_PORT + value: "{{ .Values.bot.service.nodePort }}" - name: LOG_JSON value: "1" - name: LOG_LEVEL @@ -77,7 +103,7 @@ spec: value: "{{ .Values.bot.stopIfUnhealthy }}" ports: - name: http - containerPort: {{ .Values.bot.service.port }} + containerPort: {{ .Values.bot.service.nodePort }} protocol: TCP readinessProbe: exec: @@ -87,7 +113,7 @@ spec: - | curl -s -X POST -H 'content-type: application/json' \ -d '{"jsonrpc":"2.0","method":"pxe_getNodeInfo","params":[],"id":67}' \ - 127.0.0.1:{{ .Values.bot.service.port }} > /tmp/probe_output.txt && \ + 127.0.0.1:{{ .Values.bot.service.nodePort }} > /tmp/probe_output.txt && \ cat /tmp/probe_output.txt && \ grep -q '"enr:-' /tmp/probe_output.txt initialDelaySeconds: {{ .Values.bot.readinessProbe.initialDelaySeconds }} @@ -111,8 +137,8 @@ spec: app: bot ports: - protocol: TCP - port: {{ .Values.bot.service.port }} - targetPort: {{ .Values.bot.service.targetPort }} + port: {{ .Values.bot.service.nodePort }} + targetPort: {{ .Values.bot.service.nodePort }} {{- if and (eq .Values.bot.service.type "NodePort") .Values.bot.service.nodePort }} nodePort: {{ .Values.bot.service.nodePort }} {{- end }} diff --git a/spartan/aztec-network/templates/validator.yaml b/spartan/aztec-network/templates/validator.yaml index d9c3dd9a21b..f5a2fb8ce54 100644 --- a/spartan/aztec-network/templates/validator.yaml +++ b/spartan/aztec-network/templates/validator.yaml @@ -18,17 +18,25 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: validator spec: + {{- if .Values.network.public }} + hostNetwork: true + {{- include "aztec-network.publicAntiAffinity" . | nindent 6 }} + {{- end }} + serviceAccountName: {{ include "aztec-network.fullname" . }}-node initContainers: + {{- include "aztec-network.p2pSetupContainer" . | nindent 8 }} + {{- include "aztec-network.serviceAddressSetupContainer" . | nindent 8 }} - name: wait-for-services - image: {{ .Values.images.curl.image }} + image: {{ .Values.images.aztec.image }} command: - - /bin/sh + - /bin/bash - -c - | + source /shared/config/service-addresses # First check ethereum node until curl -s -X POST -H 'Content-Type: application/json' \ -d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \ - {{ include "aztec-network.ethereumHost" . }} | grep -q reth; do + $ETHEREUM_HOST | grep -q reth; do echo "Waiting for Ethereum node..." sleep 5 done @@ -44,59 +52,47 @@ spec: if [ "{{ .Values.validator.dynamicBootNode }}" = "true" ]; then # Get the list of pod IPs for the validator service - MAX_ATTEMPTS=3 - for i in $(seq 0 $(({{ .Values.validator.replicas }} - 1))); do - PEER_IP="{{ include "aztec-network.fullname" . }}-validator-${i}.{{ include "aztec-network.fullname" . }}-validator" - echo "Checking ${PEER_IP} for /status" - for attempt in $(seq 1 $MAX_ATTEMPTS); do - if curl --silent --head --fail "http://${PEER_IP}:{{ .Values.validator.service.nodePort }}/status" > /dev/null; then - echo "Found responsive peer at ${PEER_IP}" - # the PXE has its node set to the the validator service. - # and that's all we need to know to bootstrap, - # since it will get a good node ENR from whatever node the PXE connects to. - echo "{{ include "aztec-network.pxeUrl" . }}" > /shared/pxe_url - break 2 - fi - sleep 2 - done - done - if [ ! -f /shared/pxe_url ]; then - echo "No responsive peers found after multiple attempts, exiting." - exit 1 - fi + echo "{{ include "aztec-network.pxeUrl" . }}" > /shared/pxe/pxe_url else - until curl --silent --head --fail "{{ include "aztec-network.bootNodeUrl" . }}/status" > /dev/null; do + until curl --silent --head --fail "${BOOT_NODE_HOST}/status" > /dev/null; do echo "Waiting for boot node..." sleep 5 done echo "Boot node is ready!" - echo "{{ include "aztec-network.bootNodeUrl" . }}" > /shared/pxe_url + echo "${BOOT_NODE_HOST}" > /shared/pxe/pxe_url fi volumeMounts: - - name: shared-volume - mountPath: /shared + - name: pxe-url + mountPath: /shared/pxe + - name: scripts + mountPath: /scripts + - name: config + mountPath: /shared/config - name: configure-validator-env image: "{{ .Values.images.aztec.image }}" imagePullPolicy: {{ .Values.images.aztec.pullPolicy }} command: - - "/bin/sh" + - "/bin/bash" - "-c" - | + source /shared/config/service-addresses && \ cp /scripts/configure-validator-env.sh /tmp/configure-validator-env.sh && \ chmod +x /tmp/configure-validator-env.sh && \ - /tmp/configure-validator-env.sh "$(cat /shared/pxe_url)" + /tmp/configure-validator-env.sh "$(cat /shared/pxe/pxe_url)" volumeMounts: - - name: shared-volume - mountPath: /shared + - name: contracts-env + mountPath: /shared/contracts + - name: pxe-url + mountPath: /shared/pxe - name: scripts mountPath: /scripts - name: validator-keys mountPath: /app/config readOnly: true + - name: config + mountPath: /shared/config env: - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - name: POD_NAME valueFrom: fieldRef: @@ -108,7 +104,13 @@ spec: command: - "/bin/bash" - "-c" - - "sleep 10 && source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer" + - | + sleep 10 && \ + source /shared/contracts/contracts.env && \ + source /shared/p2p/p2p-addresses && \ + source /shared/config/service-addresses && \ + env && \ + node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer startupProbe: httpGet: path: /status @@ -126,14 +128,18 @@ spec: timeoutSeconds: 30 failureThreshold: 3 volumeMounts: - - name: shared-volume - mountPath: /shared + - name: contracts-env + mountPath: /shared/contracts + - name: p2p-addresses + mountPath: /shared/p2p + - name: config + mountPath: /shared/config env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: PORT + - name: AZTEC_PORT value: "{{ .Values.validator.service.nodePort }}" - name: LOG_LEVEL value: "{{ .Values.validator.logLevel }}" @@ -141,8 +147,6 @@ spec: value: "1" - name: DEBUG value: "{{ .Values.validator.debug }}" - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - name: P2P_ENABLED value: "{{ .Values.validator.p2p.enabled }}" - name: VALIDATOR_DISABLED @@ -157,22 +161,6 @@ spec: value: "{{ .Values.validator.sequencer.enforceTimeTable }}" - name: L1_CHAIN_ID value: "{{ .Values.ethereum.chainId }}" - - name: P2P_TCP_ANNOUNCE_ADDR - {{- if .Values.validator.externalTcpHost }} - value: "{{ .Values.validator.externalTcpHost }}:{{ .Values.validator.service.p2pTcpPort }}" - {{- else }} - value: "$(POD_IP):{{ .Values.validator.service.p2pTcpPort }}" - {{- end }} - - name: P2P_UDP_ANNOUNCE_ADDR - {{- if .Values.validator.externalUdpHost }} - value: "{{ .Values.validator.externalUdpHost }}:{{ .Values.validator.service.p2pUdpPort }}" - {{- else }} - value: "$(POD_IP):{{ .Values.validator.service.p2pUdpPort }}" - {{- end }} - - name: P2P_TCP_LISTEN_ADDR - value: "0.0.0.0:{{ .Values.validator.service.p2pTcpPort }}" - - name: P2P_UDP_LISTEN_ADDR - value: "0.0.0.0:{{ .Values.validator.service.p2pUdpPort }}" - name: OTEL_RESOURCE_ATTRIBUTES value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }} - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT @@ -199,24 +187,21 @@ spec: volumes: - name: scripts configMap: - name: {{ include "aztec-network.fullname" . }}-configure-validator-env + name: {{ include "aztec-network.fullname" . }}-scripts - name: validator-keys configMap: name: {{ include "aztec-network.fullname" . }}-validator-keys - - name: shared-volume + - name: contracts-env + emptyDir: {} + - name: p2p-addresses + emptyDir: {} + - name: pxe-url + emptyDir: {} + - name: config emptyDir: {} --- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "aztec-network.fullname" . }}-configure-validator-env - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -data: - configure-validator-env.sh: | - {{ .Files.Get "files/config/config-validator-env.sh" | nindent 4 }} ---- -# Headless service for StatefulSet DNS entries +# If this is not a public network, create a headless service for StatefulSet DNS entries +{{ if not .Values.network.public }} apiVersion: v1 kind: Service metadata: @@ -238,55 +223,4 @@ spec: - port: {{ .Values.validator.service.nodePort }} name: node protocol: TCP ---- -{{if .Values.network.public }} -{{- range $i, $e := until (int .Values.validator.replicas) }} -# Service template for TCP load balancers -apiVersion: v1 -kind: Service -metadata: - name: validator-{{ $i }}-lb-tcp - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" - service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" - labels: - {{- include "aztec-network.labels" $ | nindent 4 }} -spec: - type: LoadBalancer - selector: - statefulset.kubernetes.io/pod-name: {{ include "aztec-network.fullname" $ }}-validator-{{ $i }} - {{- include "aztec-network.selectorLabels" $ | nindent 4 }} - app: validator - ports: - - port: {{ $.Values.validator.service.p2pTcpPort }} - name: p2p-tcp - protocol: TCP - - port: {{ $.Values.validator.service.nodePort }} - name: node - protocol: TCP ---- -# Service template for UDP load balancers -apiVersion: v1 -kind: Service -metadata: - name: validator-{{ $i }}-lb-udp - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" - service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" - labels: - {{- include "aztec-network.labels" $ | nindent 4 }} -spec: - type: LoadBalancer - selector: - statefulset.kubernetes.io/pod-name: {{ include "aztec-network.fullname" $ }}-validator-{{ $i }} - {{- include "aztec-network.selectorLabels" $ | nindent 4 }} - app: validator - ports: - - port: {{ $.Values.validator.service.p2pUdpPort }} - name: p2p-udp - protocol: UDP ---- -{{- end }} {{ end }} diff --git a/spartan/aztec-network/values.yaml b/spartan/aztec-network/values.yaml index d962a22d1aa..17b879a88e5 100644 --- a/spartan/aztec-network/values.yaml +++ b/spartan/aztec-network/values.yaml @@ -2,6 +2,12 @@ nameOverride: "" fullnameOverride: "" network: + # If true, pods will use host networking. + # This is to ensure that nodes are individually addressable from the outside. + # Under the current configuration, this also means that there must be a unique + # physical node in the cluster for each pod that participates in peer-to-peer. + # I.e. the sum of the number of validator, boot node, and prover nodes must be + # less than the number of physical nodes in the cluster. public: false setupL2Contracts: true @@ -29,8 +35,7 @@ aztec: epochProofClaimWindow: 13 # in L2 slots bootNode: - externalTcpHost: "" - externalUdpHost: "" + externalHost: "" replicas: 1 service: p2pTcpPort: 40400 @@ -71,8 +76,6 @@ validator: # This cannot be used when the network first starts up. # But it must be used if the boot node is killed, and the validator is restarted. dynamicBootNode: false - externalTcpHost: "" - externalUdpHost: "" replicas: 1 validatorKeys: - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 @@ -83,7 +86,7 @@ validator: p2pUdpPort: 40400 nodePort: 8080 logLevel: "debug" - debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:l2_block_stream,-aztec:world-state:database" + debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:world-state:database,-aztec:l2_block_stream*" sequencer: maxSecondsBetweenBlocks: 0 minTxsPerBlock: 1 @@ -104,9 +107,7 @@ validator: cpu: "200m" proverNode: - public: false - externalTcpHost: "" - externalUdpHost: "" + externalHost: "" replicas: 1 p2pEnabled: true service: @@ -125,15 +126,12 @@ proverNode: pxe: proverEnabled: false - externalHost: "" logLevel: "debug" proverEnable: false debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:world-state:database,-aztec:l2_block_stream*" replicas: 1 service: - port: 8080 - targetPort: 8080 - nodePort: "" + nodePort: 8081 readinessProbe: initialDelaySeconds: 5 periodSeconds: 10 @@ -164,9 +162,7 @@ bot: stopIfUnhealthy: true service: type: ClusterIP - port: 8080 - targetPort: 8080 - nodePort: "" + nodePort: 8082 readinessProbe: initialDelaySeconds: 5 periodSeconds: 10 @@ -205,13 +201,18 @@ ethereum: storage: "80Gi" proverAgent: + service: + nodePort: 8083 enabled: true replicas: 1 + logLevel: "debug" debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:world-state:database,-aztec:l2_block_stream*" realProofs: false concurrency: 1 bb: hardwareConcurrency: "" + nodeSelector: {} + resources: {} jobs: deployL1Verifier: diff --git a/yarn-project/end-to-end/scripts/network_test.sh b/yarn-project/end-to-end/scripts/network_test.sh index 8c9890f4657..6789c4a75b8 100755 --- a/yarn-project/end-to-end/scripts/network_test.sh +++ b/yarn-project/end-to-end/scripts/network_test.sh @@ -167,7 +167,7 @@ docker run --rm --network=host \ -e SPARTAN_DIR="/usr/src/spartan" \ -e NAMESPACE="$NAMESPACE" \ -e HOST_PXE_PORT=$PXE_PORT \ - -e CONTAINER_PXE_PORT=8080 \ + -e CONTAINER_PXE_PORT=8081 \ -e HOST_ETHEREUM_PORT=$ANVIL_PORT \ -e CONTAINER_ETHEREUM_PORT=8545 \ -e DEBUG="aztec:*" \ diff --git a/yarn-project/end-to-end/src/spartan/4epochs.test.ts b/yarn-project/end-to-end/src/spartan/4epochs.test.ts index 29f24e42f56..feef5c9f243 100644 --- a/yarn-project/end-to-end/src/spartan/4epochs.test.ts +++ b/yarn-project/end-to-end/src/spartan/4epochs.test.ts @@ -30,13 +30,13 @@ describe('token transfer test', () => { beforeAll(async () => { if (isK8sConfig(config)) { await startPortForward({ - resource: 'svc/spartan-aztec-network-pxe', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, namespace: config.NAMESPACE, containerPort: config.CONTAINER_PXE_PORT, hostPort: config.HOST_PXE_PORT, }); await startPortForward({ - resource: 'svc/spartan-aztec-network-ethereum', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-ethereum`, namespace: config.NAMESPACE, containerPort: config.CONTAINER_ETHEREUM_PORT, hostPort: config.HOST_ETHEREUM_PORT, diff --git a/yarn-project/end-to-end/src/spartan/gating-passive.test.ts b/yarn-project/end-to-end/src/spartan/gating-passive.test.ts index 03726310982..8623027e7de 100644 --- a/yarn-project/end-to-end/src/spartan/gating-passive.test.ts +++ b/yarn-project/end-to-end/src/spartan/gating-passive.test.ts @@ -41,13 +41,13 @@ describe('a test that passively observes the network in the presence of network it('survives network chaos', async () => { await startPortForward({ - resource: 'svc/spartan-aztec-network-pxe', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, namespace: NAMESPACE, containerPort: CONTAINER_PXE_PORT, hostPort: HOST_PXE_PORT, }); await startPortForward({ - resource: 'svc/spartan-aztec-network-ethereum', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-ethereum`, namespace: NAMESPACE, containerPort: CONTAINER_ETHEREUM_PORT, hostPort: HOST_ETHEREUM_PORT, diff --git a/yarn-project/end-to-end/src/spartan/proving.test.ts b/yarn-project/end-to-end/src/spartan/proving.test.ts index fa1d1e73807..8681f17601c 100644 --- a/yarn-project/end-to-end/src/spartan/proving.test.ts +++ b/yarn-project/end-to-end/src/spartan/proving.test.ts @@ -19,7 +19,7 @@ describe('proving test', () => { let PXE_URL; if (isK8sConfig(config)) { proc = await startPortForward({ - resource: 'svc/spartan-aztec-network-pxe', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, namespace: config.NAMESPACE, containerPort: config.CONTAINER_PXE_PORT, hostPort: config.HOST_PXE_PORT, diff --git a/yarn-project/end-to-end/src/spartan/reorg.test.ts b/yarn-project/end-to-end/src/spartan/reorg.test.ts index 8421e7387c5..c315fe05def 100644 --- a/yarn-project/end-to-end/src/spartan/reorg.test.ts +++ b/yarn-project/end-to-end/src/spartan/reorg.test.ts @@ -47,13 +47,13 @@ describe('reorg test', () => { it('survives a reorg', async () => { await startPortForward({ - resource: 'svc/spartan-aztec-network-pxe', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, namespace: NAMESPACE, containerPort: CONTAINER_PXE_PORT, hostPort: HOST_PXE_PORT, }); await startPortForward({ - resource: 'svc/spartan-aztec-network-ethereum', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-ethereum`, namespace: NAMESPACE, containerPort: CONTAINER_ETHEREUM_PORT, hostPort: HOST_ETHEREUM_PORT, @@ -99,7 +99,7 @@ describe('reorg test', () => { await waitForResourceByLabel({ resource: 'pods', namespace: NAMESPACE, label: 'app=pxe' }); await sleep(30 * 1000); await startPortForward({ - resource: 'svc/spartan-aztec-network-pxe', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, namespace: NAMESPACE, containerPort: CONTAINER_PXE_PORT, hostPort: HOST_PXE_PORT, diff --git a/yarn-project/end-to-end/src/spartan/smoke.test.ts b/yarn-project/end-to-end/src/spartan/smoke.test.ts index 43a67dc9d12..dc47f4f97f8 100644 --- a/yarn-project/end-to-end/src/spartan/smoke.test.ts +++ b/yarn-project/end-to-end/src/spartan/smoke.test.ts @@ -18,7 +18,7 @@ describe('smoke test', () => { let PXE_URL; if (isK8sConfig(config)) { await startPortForward({ - resource: 'svc/spartan-aztec-network-pxe', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, namespace: config.NAMESPACE, containerPort: config.CONTAINER_PXE_PORT, hostPort: config.HOST_PXE_PORT, diff --git a/yarn-project/end-to-end/src/spartan/transfer.test.ts b/yarn-project/end-to-end/src/spartan/transfer.test.ts index e82f0518fb0..a1a9d7aea9a 100644 --- a/yarn-project/end-to-end/src/spartan/transfer.test.ts +++ b/yarn-project/end-to-end/src/spartan/transfer.test.ts @@ -23,7 +23,7 @@ describe('token transfer test', () => { let PXE_URL; if (isK8sConfig(config)) { await startPortForward({ - resource: 'svc/spartan-aztec-network-pxe', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, namespace: config.NAMESPACE, containerPort: config.CONTAINER_PXE_PORT, hostPort: config.HOST_PXE_PORT,