chore: Greater stability at 1TPS (#10981)

AztecProtocol · Jan 9, 2025 · 1c23662 · 1c23662
1 parent 90cd9d2
commit 1c23662
Show file tree

Hide file tree

Showing 46 changed files with 1,641 additions and 429 deletions.
diff --git a/boxes/yarn.lock b/boxes/yarn.lock
@@ -78,7 +78,7 @@ __metadata:
     "@aztec/ethereum": "workspace:^"
     "@aztec/foundation": "workspace:^"
     idb: "npm:^8.0.0"
-    lmdb: "npm:^3.0.6"
+    lmdb: "npm:^3.2.0"
   languageName: node
   linkType: soft
 
@@ -1553,44 +1553,44 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@lmdb/lmdb-darwin-arm64@npm:3.1.6":
-  version: 3.1.6
-  resolution: "@lmdb/lmdb-darwin-arm64@npm:3.1.6"
+"@lmdb/lmdb-darwin-arm64@npm:3.2.1":
+  version: 3.2.1
+  resolution: "@lmdb/lmdb-darwin-arm64@npm:3.2.1"
   conditions: os=darwin & cpu=arm64
   languageName: node
   linkType: hard
 
-"@lmdb/lmdb-darwin-x64@npm:3.1.6":
-  version: 3.1.6
-  resolution: "@lmdb/lmdb-darwin-x64@npm:3.1.6"
+"@lmdb/lmdb-darwin-x64@npm:3.2.1":
+  version: 3.2.1
+  resolution: "@lmdb/lmdb-darwin-x64@npm:3.2.1"
   conditions: os=darwin & cpu=x64
   languageName: node
   linkType: hard
 
-"@lmdb/lmdb-linux-arm64@npm:3.1.6":
-  version: 3.1.6
-  resolution: "@lmdb/lmdb-linux-arm64@npm:3.1.6"
+"@lmdb/lmdb-linux-arm64@npm:3.2.1":
+  version: 3.2.1
+  resolution: "@lmdb/lmdb-linux-arm64@npm:3.2.1"
   conditions: os=linux & cpu=arm64
   languageName: node
   linkType: hard
 
-"@lmdb/lmdb-linux-arm@npm:3.1.6":
-  version: 3.1.6
-  resolution: "@lmdb/lmdb-linux-arm@npm:3.1.6"
+"@lmdb/lmdb-linux-arm@npm:3.2.1":
+  version: 3.2.1
+  resolution: "@lmdb/lmdb-linux-arm@npm:3.2.1"
   conditions: os=linux & cpu=arm
   languageName: node
   linkType: hard
 
-"@lmdb/lmdb-linux-x64@npm:3.1.6":
-  version: 3.1.6
-  resolution: "@lmdb/lmdb-linux-x64@npm:3.1.6"
+"@lmdb/lmdb-linux-x64@npm:3.2.1":
+  version: 3.2.1
+  resolution: "@lmdb/lmdb-linux-x64@npm:3.2.1"
   conditions: os=linux & cpu=x64
   languageName: node
   linkType: hard
 
-"@lmdb/lmdb-win32-x64@npm:3.1.6":
-  version: 3.1.6
-  resolution: "@lmdb/lmdb-win32-x64@npm:3.1.6"
+"@lmdb/lmdb-win32-x64@npm:3.2.1":
+  version: 3.2.1
+  resolution: "@lmdb/lmdb-win32-x64@npm:3.2.1"
   conditions: os=win32 & cpu=x64
   languageName: node
   linkType: hard
@@ -8398,16 +8398,16 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lmdb@npm:^3.0.6":
-  version: 3.1.6
-  resolution: "lmdb@npm:3.1.6"
-  dependencies:
-    "@lmdb/lmdb-darwin-arm64": "npm:3.1.6"
-    "@lmdb/lmdb-darwin-x64": "npm:3.1.6"
-    "@lmdb/lmdb-linux-arm": "npm:3.1.6"
-    "@lmdb/lmdb-linux-arm64": "npm:3.1.6"
-    "@lmdb/lmdb-linux-x64": "npm:3.1.6"
-    "@lmdb/lmdb-win32-x64": "npm:3.1.6"
+"lmdb@npm:^3.2.0":
+  version: 3.2.1
+  resolution: "lmdb@npm:3.2.1"
+  dependencies:
+    "@lmdb/lmdb-darwin-arm64": "npm:3.2.1"
+    "@lmdb/lmdb-darwin-x64": "npm:3.2.1"
+    "@lmdb/lmdb-linux-arm": "npm:3.2.1"
+    "@lmdb/lmdb-linux-arm64": "npm:3.2.1"
+    "@lmdb/lmdb-linux-x64": "npm:3.2.1"
+    "@lmdb/lmdb-win32-x64": "npm:3.2.1"
     msgpackr: "npm:^1.11.2"
     node-addon-api: "npm:^6.1.0"
     node-gyp: "npm:latest"
@@ -8429,7 +8429,7 @@ __metadata:
       optional: true
   bin:
     download-lmdb-prebuilds: bin/download-prebuilds.js
-  checksum: 10c0/081804f72aab6eb0f712654e3bbb2d454dd455bbfe09f223e10728971f201cfc166d4d6dd6a3099aabf79e4fd62e9c2a5eb9117bd5f2153ec5a419333f69a338
+  checksum: 10c0/cccf17b95a821c56d3eef87abbdd7c30e4bc3147fbdd1cae44a5ba8f1977dbb3177021cfda9b7e8a293fc53fe78fc81979ce032885b777d3bec556c255c25724
   languageName: node
   linkType: hard
 

diff --git a/cspell.json b/cspell.json
@@ -11,8 +11,8 @@
     "asyncify",
     "auditability",
     "authwit",
-    "authwits",
     "authwitness",
+    "authwits",
     "Automine",
     "autonat",
     "autorun",
@@ -125,6 +125,7 @@
     "ierc",
     "indexeddb",
     "initialise",
+    "initialised",
     "initialising",
     "interruptible",
     "isequal",

diff --git a/spartan/aztec-network/templates/_helpers.tpl b/spartan/aztec-network/templates/_helpers.tpl
@@ -183,3 +183,8 @@ affinity:
         topologyKey: "kubernetes.io/hostname"
         namespaceSelector: {}
 {{- end -}}
+
+{{- define "aztec-network.gcpLocalSsd" -}}
+nodeSelector:
+  cloud.google.com/gke-ephemeral-storage-local-ssd: "true"
+{{- end -}}
diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml
@@ -11,6 +11,7 @@ spec:
     matchLabels:
       {{- include "aztec-network.selectorLabels" . | nindent 6 }}
       app: boot-node
+  {{- if not .Values.storage.localSsd }}
   volumeClaimTemplates:
     - metadata:
         name: boot-node-data
@@ -19,12 +20,16 @@ spec:
         resources:
           requests:
             storage: {{ .Values.bootNode.storageSize }}
+  {{- end }}
   template:
     metadata:
       labels:
         {{- include "aztec-network.selectorLabels" . | nindent 8 }}
         app: boot-node
     spec:
+      {{- if .Values.storage.localSsd }}
+      {{- include "aztec-network.gcpLocalSsd" . | nindent 6 }}
+      {{- end }}
       dnsPolicy: ClusterFirstWithHostNet
       {{- if .Values.network.public }}
       hostNetwork: true
@@ -174,6 +179,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
+            - name: NODE_OPTIONS
+              value: "--max-old-space-size={{ .Values.bootNode.maxOldSpaceSize}}"
             - name: AZTEC_PORT
               value: "{{ .Values.bootNode.service.nodePort }}"
             - name: LOG_LEVEL
@@ -220,6 +227,12 @@ spec:
               value: {{ .Values.bootNode.viemPollingInterval | quote }}
             - name: PEER_ID_PRIVATE_KEY
               value: "{{ .Values.bootNode.peerIdPrivateKey }}"
+            - name: DATA_DIRECTORY
+              value: "{{ .Values.bootNode.dataDir }}"
+            - name: DATA_STORE_MAP_SIZE_KB
+              value: "{{ .Values.storage.dataStoreMapSize }}"
+            - name: WS_DB_MAP_SIZE_KB
+              value: "{{ .Values.storage.worldStateMapSize }}"
           ports:
             - containerPort: {{ .Values.bootNode.service.nodePort }}
             - containerPort: {{ .Values.bootNode.service.p2pTcpPort }}
@@ -232,9 +245,14 @@ spec:
           emptyDir: {}
         - name: config
           emptyDir: {}
+      {{- if .Values.storage.localSsd }}
+        - name: boot-node-data
+          emptyDir: {}
+      {{ else }}
         - name: boot-node-data
           persistentVolumeClaim:
             claimName: boot-node-data
+      {{- end }}
       {{- if .Values.bootNode.deployContracts }}
         - name: scripts
           configMap:

diff --git a/spartan/aztec-network/templates/prover-broker.yaml b/spartan/aztec-network/templates/prover-broker.yaml
@@ -1,6 +1,6 @@
 {{- if .Values.proverBroker.enabled }}
 apiVersion: apps/v1
-kind: ReplicaSet
+kind: StatefulSet
 metadata:
   name: {{ include "aztec-network.fullname" . }}-prover-broker
   labels:
@@ -11,13 +11,26 @@ spec:
     matchLabels:
       {{- include "aztec-network.selectorLabels" . | nindent 6 }}
       app: prover-broker
+  {{- if not .Values.storage.localSsd }}
+  volumeClaimTemplates:
+    - metadata:
+        name: prover-broker-data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: {{ .Values.proverBroker.storageSize }}
+  {{- end }}
   template:
     metadata:
       labels:
         {{- include "aztec-network.selectorLabels" . | nindent 8 }}
         app: prover-broker
     spec:
       serviceAccountName: {{ include "aztec-network.fullname" . }}-node
+      {{- if .Values.storage.localSsd }}
+      {{- include "aztec-network.gcpLocalSsd" . | nindent 6 }}
+      {{- end }}
       {{- if .Values.network.public }}
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
@@ -26,6 +39,8 @@ spec:
       volumes:
         - name: config
           emptyDir: {}
+        - name: prover-broker-data
+          emptyDir: {}
         - name: scripts
           configMap:
             name: {{ include "aztec-network.fullname" . }}-scripts
@@ -55,6 +70,8 @@ spec:
           volumeMounts:
             - name: config
               mountPath: /shared/config
+            - name: prover-broker-data
+              mountPath: {{ .Values.proverBroker.dataDir }}
           command:
             - "/bin/bash"
             - "-c"
@@ -70,10 +87,14 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
+            - name: NODE_OPTIONS
+              value: "--max-old-space-size={{ .Values.proverBroker.maxOldSpaceSize}}"
             - name: AZTEC_PORT
               value: "{{ .Values.proverBroker.service.nodePort }}"
             - name: LOG_LEVEL
               value: "{{ .Values.proverBroker.logLevel }}"
+            - name: DATA_DIRECTORY
+              value: "{{ .Values.proverBroker.dataDir }}"
             - name: LOG_JSON
               value: "1"
             - name: PROVER_BROKER_POLL_INTERVAL_MS
@@ -83,11 +104,27 @@ spec:
             - name: PROVER_BROKER_JOB_MAX_RETRIES
               value: "{{ .Values.proverBroker.jobMaxRetries }}"
             - name: PROVER_BROKER_DATA_DIRECTORY
-              value: "{{ .Values.proverBroker.dataDirectory }}"
+              value: "{{ .Values.proverBroker.dataDir }}"
+            - name: PROVER_BROKER_DATA_MAP_SIZE_KB
+              value: "{{ .Values.storage.dataStoreMapSize }}"
             - name: OTEL_RESOURCE_ATTRIBUTES
               value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
           resources:
             {{- toYaml .Values.proverBroker.resources | nindent 12 }}
+      volumes:
+        - name: scripts
+          configMap:
+            name: {{ include "aztec-network.fullname" . }}-scripts
+        - name: config
+          emptyDir: {}
+      {{- if .Values.storage.localSsd }}
+        - name: prover-broker-data
+          emptyDir: {}
+      {{ else }}
+        - name: prover-broker-data
+          persistentVolumeClaim:
+            claimName: prover-broker-data
+      {{- end }}
 {{- end }}
 ---
 # Headless service for StatefulSet DNS entries

diff --git a/spartan/aztec-network/templates/prover-node.yaml b/spartan/aztec-network/templates/prover-node.yaml
@@ -11,12 +11,25 @@ spec:
     matchLabels:
       {{- include "aztec-network.selectorLabels" . | nindent 6 }}
       app: prover-node
+  {{- if not .Values.storage.localSsd }}
+  volumeClaimTemplates:
+    - metadata:
+        name: prover-node-data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: {{ .Values.proverNode.storageSize }}
+  {{- end }}
   template:
     metadata:
       labels:
         {{- include "aztec-network.selectorLabels" . | nindent 8 }}
         app: prover-node
     spec:
+      {{- if .Values.storage.localSsd }}
+      {{- include "aztec-network.gcpLocalSsd" . | nindent 6 }}
+      {{- end }}
       {{- if .Values.network.public }}
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
@@ -106,6 +119,8 @@ spec:
               mountPath: /shared/p2p
             - name: config
               mountPath: /shared/config
+            - name: prover-node-data
+              mountPath: {{ .Values.proverNode.dataDir }}
           env:
             - name: K8S_POD_UID
               valueFrom:
@@ -119,6 +134,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: NODE_OPTIONS
+              value: "--max-old-space-size={{ .Values.proverNode.maxOldSpaceSize}}"
             - name: AZTEC_PORT
               value: "{{ .Values.proverNode.service.nodePort }}"
             - name: LOG_LEVEL
@@ -171,6 +188,12 @@ spec:
               value: "{{ .Values.aztec.epochProofClaimWindow }}"
             - name: PROVER_VIEM_POLLING_INTERVAL_MS
               value: {{ .Values.proverNode.viemPollingInterval | quote }}
+            - name: DATA_DIRECTORY
+              value: "{{ .Values.proverNode.dataDir }}"
+            - name: DATA_STORE_MAP_SIZE_KB
+              value: "{{ .Values.storage.dataStoreMapSize }}"
+            - name: WS_DB_MAP_SIZE_KB
+              value: "{{ .Values.storage.worldStateMapSize }}"
           ports:
             - containerPort: {{ .Values.proverNode.service.nodePort }}
             - containerPort: {{ .Values.proverNode.service.p2pTcpPort }}
@@ -188,16 +211,14 @@ spec:
           emptyDir: {}
         - name: config
           emptyDir: {}
-  volumeClaimTemplates:
-  - metadata:
-      name: shared-volume
-      labels:
-        {{- include "aztec-network.labels" . | nindent 8 }}
-    spec:
-      accessModes: ["ReadWriteOnce"]
-      resources:
-        requests:
-          storage: {{ .Values.proverNode.storage }}
+      {{- if .Values.storage.localSsd }}
+        - name: prover-node-data
+          emptyDir: {}
+      {{ else }}
+        - name: prover-node-data
+          persistentVolumeClaim:
+            claimName: prover-node-data
+      {{- end }}
 {{if not .Values.network.public }}
 ---
 apiVersion: v1

diff --git a/spartan/aztec-network/templates/reth.yaml b/spartan/aztec-network/templates/reth.yaml
@@ -143,7 +143,7 @@ spec:
   accessModes: ["ReadWriteOnce"]
   resources:
     requests:
-      storage: {{ .Values.ethereum.storage }}
+      storage: {{ .Values.ethereum.storageSize }}
 {{- end }}
 ---
 {{ end }}