mainnet: change layout

README.md

@@ -8,8 +8,8 @@ These are [Beacon API](https://ethereum.github.io/beacon-APIs/) endpoints intend
 
 | Endpoint                                        | Host                                 |
 |-------------------------------------------------|--------------------------------------|
-| http://unstable.mainnet.beacon-api.nimbus.team/ | `linux-01.ih-eu-mda1.nimbus.mainnet` |
-| http://testing.mainnet.beacon-api.nimbus.team/  | `linux-02.ih-eu-mda1.nimbus.mainnet` |
+| http://unstable.mainnet.beacon-api.nimbus.team/ | `nel-01.ih-eu-mda1.nimbus.mainnet` |
+| http://testing.mainnet.beacon-api.nimbus.team/  | `nel-02.ih-eu-mda1.nimbus.mainnet` |
 | http://unstable.sepolia.beacon-api.nimbus.team/ | `linux-01.ih-eu-mda1.nimbus.sepolia` |
 | http://unstable.holesky.beacon-api.nimbus.team/ | `geth-01.ih-eu-mda1.nimbus.holesky`  |
 | http://testing.holesky.beacon-api.nimbus.team/  | `geth-02.ih-eu-mda1.nimbus.holesky`  |
@@ -20,7 +20,7 @@ There are also archives of ERA files:
 
 | Endpoint                          | Host                                 |
 |-----------------------------------|--------------------------------------|
-| https://mainnet.era.nimbus.team/  | `linux-03.ih-eu-mda1.nimbus.mainnet` |
+| https://mainnet.era.nimbus.team/  | `nel-03.ih-eu-mda1.nimbus.mainnet` |
 | https://sepolia.era.nimbus.team/  | `linux-01.ih-eu-mda1.nimbus.sepolia` |
 | https://sepolia.era1.nimbus.team/ | `linux-01.ih-eu-mda1.nimbus.sepolia` |
 | https://holesky.era.nimbus.team/  | `geth-01.ih-eu-mda1.nimbus.holesky`  |

ansible/era.yml

@@ -12,7 +12,7 @@
 - name: Configure ERA files hosting
   become: true
   hosts:
-    - linux-03.ih-eu-mda1.nimbus.mainnet
+    - nel-03.ih-eu-mda1.nimbus.mainnet
     - linux-01.ih-eu-mda1.nimbus.sepolia
     - geth-01.ih-eu-mda1.nimbus.holesky
   roles:

ansible/group_vars/nimbus.mainnet.yml

@@ -6,42 +6,118 @@ bootstrap__extra_volume_bind_path: null
 # SWAP
 swap_file_size_mb: 2048
 
-# Go-Ethereum
-geth_service_name: 'geth-{{ geth_network_name }}'
+# Ports used for all 3 EL nodes as they are mutually exclusive.
+exec_layer_p2p_port:     '{{ 30303 + 3*idx|int + 1 }}'
+exec_layer_rpc_port:     '{{ 8545  +   idx|int + 1 }}'
+exec_layer_authrpc_port: '{{ 8551  +   idx|int + 1 }}'
+exec_layer_metrics_port: '{{ 6060  +   idx|int + 1 }}'
+
+# Go-Ethereum ------------------------------------------------------------------
+
+geth_service_name: 'geth-{{ geth_network_name }}-{{ node.branch | mandatory }}'
 geth_service_path: '/docker/{{ geth_service_name }}'
 geth_network_name: 'mainnet'
 geth_cont_name: '{{ geth_service_name }}-node'
 geth_cont_vol: '{{ geth_service_path }}/node'
 geth_sync_mode: 'snap'
 geth_log_level_name: 'info'
+# Geth auth & JWT token
 geth_account_pass: '{{lookup("bitwarden", "nimbus/geth", field="password")}}'
-geth_authrpc_jwtsecret: '{{lookup("bitwarden", "nimbus/jwt-token")}}'
-# Memory settings¬
+geth_authrpc_jwtsecret: '{{ beacon_node_exec_layer_jwt_secret }}'
+# Memory settings
+# TODO: keep or reduce?
 geth_cont_mem_ratio: 0.15
 geth_cache_size: '{{ (ansible_memtotal_mb * 0.05|float) | int }}'
 # Ports
-geth_port: 30303
-geth_rpc_addr: '0.0.0.0'
-geth_rpc_port: 8545
-geth_authrpc_addr: '127.0.0.1'
-geth_authrpc_port: 8551
-# Geth metrics¬
+geth_rpc_addr:        '0.0.0.0'
+geth_authrpc_addr:    '127.0.0.1'
+geth_port:            '{{ exec_layer_p2p_port }}'
+geth_rpc_port:        '{{ exec_layer_rpc_port }}'
+geth_authrpc_port:    '{{ exec_layer_authrpc_port }}'
+geth_metrics_port:    '{{ exec_layer_metrics_port }}'
+# Geth metrics
 geth_expo_service_name: '{{ geth_service_name }}'
 geth_expo_source_cont_name: '{{ geth_cont_name }}'
 geth_expo_source_data_path: '{{ geth_cont_vol }}/data'
-geth_expo_cont_port: 9400
+geth_expo_cont_port:  '{{ 9400 + (idx|int) + 1 }}'
+
+# Erigon -----------------------------------------------------------------------
+
+erigon_network_name: 'holesky'
+erigon_service_name: 'erigon-{{ geth_network_name }}-{{ node.branch | mandatory }}'
+erigon_prune: 'hrtc' # TODO: Possibly wrong, verify.
+erigon_log_level: 'info'
+erigon_cont_mem_ratio: 0.15
+erigon_max_peers: 20
+erigon_miner_enabled: true
+# TODO: whats that?
+erigon_mining_private_key: '{{lookup("bitwarden", "nimbus/wallet/testnets", field="private-key")}}'
+# Ports
+erigon_metrics_enabled: true
+erigon_rpc_addr: '0.0.0.0'
+erigon_authrpc_addr: '0.0.0.0'
+erigon_port:              '{{ exec_layer_p2p_port }}'
+erigon_p2p_allowed_ports: ['{{ erigon_port }}', '{{ erigon_port|int + 1 }}']
+erigon_rpc_port:          '{{ exec_layer_rpc_port }}'
+erigon_metrics_port:      '{{ exec_layer_metrics_port }}'
+erigon_authrpc_port:      '{{ exec_layer_authrpc_port }}'
+erigon_authrpc_jwtsecret: '{{ beacon_node_exec_layer_jwt_secret }}'
+
+# Nimbus ETH1 ------------------------------------------------------------------
+
+nimbus_eth1_service_name: 'nel-{{ geth_network_name }}-{{ node.branch | mandatory }}'
+nimbus_eth1_repo_branch: 'master'
+nimbus_eth1_network: 'mainnet'
+nimbus_eth1_max_peers: 160
+nimbus_eth1_log_level: 'DEBUG'
+nimbus_eth1_service_path: '/docker/{{ nimbus_eth1_service_name }}'
+nimbus_eth1_engine_http_enabled: true
+nimbus_eth1_engine_ws_enabled: true
+nimbus_eth1_era_dir: '{{ nimbus_era_files_timer_path }}'
+nimbus_eth1_era1_dir: '/docker/era1'
+# Ports
+nimbus_eth1_listening_port: '{{ exec_layer_p2p_port }}'
+nimbus_eth1_discovery_port: '{{ exec_layer_p2p_port }}'
+nimbus_eth1_http_port:      '{{ exec_layer_rpc_port }}'
+nimbus_eth1_metrics_port:   '{{ exec_layer_metrics_port }}'
+nimbus_eth1_metrics_address: '0.0.0.0'
+# API secert
+nimbus_eth1_jwt_secret: '{{ beacon_node_exec_layer_jwt_secret }}'
+
+# Beacon Nodes -----------------------------------------------------------------
 
-# Nimbus Beacon Node
 beacon_node_network: 'mainnet'
+beacon_node_service_name: 'beacon-node-{{ beacon_node_network }}-{{ node.branch | mandatory }}'
+beacon_node_service_path: '/data/{{ beacon_node_service_name }}'
+beacon_node_era_dir_path: '{{ nimbus_era_files_timer_path }}'
 beacon_node_repo_branch: '{{ node_name_to_branch_map.get(node.branch, node.branch) }}'
+# Tuning
+beacon_node_history_retention: '{{ "archive" if node.get("public_api") else node.get("history", "prune") }}'
+beacon_node_extra_flags: '{{ node.extra_flags if node.extra_flags is defined else {} }}'
+beacon_node_cores_per_node: '{{ (ansible_processor_vcpus / nodes_layout[inventory_hostname]|length) | round(0, "ceil") | int }}'
+beacon_node_threads: '{{ (node.branch == "libp2p") | ternary(1, beacon_node_cores_per_node) }}'
 # We map short names to branches to avoid too long service names.
 node_name_to_branch_map:
   libp2p: 'nim-libp2p-auto-bump-unstable'
-# TODO Drop this once all nodes have an number.
-service_number: '{{ node.get("num", False) | ternary(("-%02d"|format(node.get("num", 0))), "") }}'
-beacon_node_service_name: 'beacon-node-{{ beacon_node_network }}-{{ node.branch | mandatory }}{{ service_number }}'
-beacon_node_service_path: '/data/{{ beacon_node_service_name }}'
-beacon_node_era_dir_path: '{{ nimbus_era_files_timer_path }}'
+# Builds
+beacon_node_update_build_targets: ['nimbus_beacon_node', 'ncli_db']
+beacon_node_update_build: '{{ beacon_node_repo_branch != "stable" }}'
+beacon_node_update_frequency: '*-*-* {{ "%02d" | format(idx * 2) }}:00:00'
+# Monitoring
+beacon_node_validator_monitor_auto: true
+beacon_node_validator_monitor_details: '{{ node.public_api is not defined or not node.public_api }}'
+# Execution layer Enginer API
+beacon_node_exec_layer_jwt_secret: '{{lookup("bitwarden", "nimbus/jwt-token")}}'
+beacon_node_exec_layer_urls_local: ['http://localhost:{{ exec_layer_authrpc_port }}']
+beacon_node_exec_layer_urls: '{{ beacon_node_exec_layer_urls_local if node.get("el") else [] }}'
+# TODO try this
+#beacon_node_exec_layer_urls: '{{ ["http://localhost:{{ exec_layer_authrpc_port }}"] if node.get("el") else [] }}'
+# Light client data
+beacon_node_light_client_data_enabled: '{{ (node.public_api is defined and node.public_api) }}'
+beacon_node_light_client_data_serve: true
+beacon_node_light_client_data_import_mode: 'full'
+# Mainnet validators run on a separate fleet.
+beacon_node_dist_validators_enabled: false
 # Ports
 beacon_node_discovery_port: '{{ 9000 + idx }}'
 beacon_node_listening_port: '{{ 9000 + idx }}'
@@ -52,36 +128,11 @@ beacon_node_rest_address:   '0.0.0.0'
 beacon_node_max_peers: '{{ node.get("max_peers", 320) }}'
 # Firewall
 beacon_node_firewall_libp2p_open: '{{ node.get("open_libp2p_ports", true) }}'
-# Builds
-beacon_node_update_build_targets: ['nimbus_beacon_node', 'ncli_db']
-beacon_node_update_build: '{{ beacon_node_repo_branch != "stable" }}'
-beacon_node_update_frequency: '*-*-* {{ "%02d" | format(idx * 2) }}:00:00'
-# Tuning
-beacon_node_cores_per_node: '{{ (ansible_processor_vcpus / nodes_layout[inventory_hostname]|length) | round(0, "ceil") | int }}'
-beacon_node_threads: '{{ (node.branch == "libp2p") | ternary(1, beacon_node_cores_per_node) }}'
+# Excellent stress test and good service to the community.
+beacon_node_subscribe_all: true
 # FIXME: Temporary test to debug REST API timeout issues.
 # https://github.com/status-im/nimbus-eth2/issues/5838
 #beacon_node_rest_allow_origin: '{{ "*" if node.get("public_api") else "" }}'
-beacon_node_history_retention: '{{ "archive" if node.get("public_api") else node.get("history", "prune") }}'
-# Monitoring
-beacon_node_validator_monitor_auto: true
-beacon_node_validator_monitor_details: '{{ node.public_api is not defined or not node.public_api }}'
-# Mainnet validators run on a separate fleet.
-beacon_node_dist_validators_enabled: false
-# Excellent stress test and good service to the community.
-beacon_node_subscribe_all: true
-beacon_node_extra_flags: '{{ node.extra_flags if node.extra_flags is defined else {} }}'
-# Light client data
-beacon_node_light_client_data_enabled: '{{ (node.public_api is defined and node.public_api) }}'
-beacon_node_light_client_data_serve: true
-beacon_node_light_client_data_import_mode: 'full'
-# Execution layer Enginer API
-beacon_node_exec_layer_jwt_secret: '{{ geth_authrpc_jwtsecret }}'
-beacon_node_exec_layer_urls_local_geth: ['http://localhost:{{ geth_authrpc_port|string }}']
-beacon_node_exec_layer_urls_all: '{{ beacon_node_exec_layer_urls_discovered_geth + beacon_node_exec_layer_urls_local_geth }}'
-beacon_node_exec_layer_urls: |-
-  {{ beacon_node_exec_layer_urls_all if inventory_hostname_short == "linux-01"
-    else (idx % 2 == 0) | ternary(beacon_node_exec_layer_urls_all, []) }}
 
 # Reduce Consul alerts sensitivity
 beacon_node_consul_check_disabled: '{{ node.get("public_api", false) }}'
@@ -113,21 +164,24 @@ nimbus_era_files_nclidb_path: '/data/beacon-node-{{ beacon_node_network }}-stabl
 bootstrap__firewall_nftables: true
 
 # Open Ports
+#host_el_type: '{{ ansible_hostname|split("-")|first }}'
+host_el_type: 'EL'
 open_ports_list:
   nginx:
     - { port: '443', comment: 'Nginx' }
   smart-metrics:
     - { port: '{{ smart_metrics_listen_port }}', comment: 'SMART Metrics', ipset: 'metrics.hq', iifname: 'wg0' }
-  geth-node:
-    - { port: '{{ geth_port }}',           comment: 'Go-Ethereum libp2p',    protocol: 'tcp'                                }
-    - { port: '{{ geth_port }}',           comment: 'Go-Ethereum discovery', protocol: 'udp'                                }
-    - { port: '{{ geth_expo_cont_port }}', comment: 'Go-Ethereum Exporter',  ipset: 'metrics.hq',            iifname: 'wg0' }
-    - { port: '{{ geth_authrpc_port }}',   comment: 'Go-Ethereum AuthRPC',   ipset: '{{ env }}.{{ stage }}', iifname: 'wg0' }
-    - { port: '{{ geth_rpc_port }}',       comment: 'RPC for Portal Bridge', ipset: 'nimbus.fluffy',         iifname: 'wg0' }
+  exec-node:
+    - { port: '30304-30316', comment: '{{ host_el_type }} LibP2P',    protocol: 'tcp'                     }
+    - { port: '30304-30316', comment: '{{ host_el_type }} Discovery', protocol: 'udp'                     }
+    - { port: '6061-6064',   comment: '{{ host_el_type }} Metrics',   ipset: 'metrics.hq', iifname: 'wg0' }
+    - { port: '9401-9404',   comment: '{{ host_el_type }} Exporter',  ipset: 'metrics.hq', iifname: 'wg0' }
+    - { port: '8552-8555',   comment: 'Go-Ethereum AuthRPC',   ipset: '{{ env }}.{{ stage }}', iifname: 'wg0' }
+    - { port: '8546-8449',   comment: 'RPC for Portal Bridge', ipset: 'nimbus.fluffy',         iifname: 'wg0' }
   beacon-node:
-    - { port: '9000-9005', comment: 'Beacon Node libp2p',    protocol: 'tcp'                                    }
-    - { port: '9000-9005', comment: 'Beacon Node discovery', protocol: 'udp'                                    }
-    - { port: '9200-9205', comment: 'Beacon Node Metrics',   ipset: 'metrics.hq',                iifname: 'wg0' }
-    - { port: '9300-9305', comment: 'Beacon Node REST API',  ipset: '{{ env }}.{{ stage }}',     iifname: 'wg0' }
+    - { port: '9001-9005', comment: 'Beacon Node libp2p',    protocol: 'tcp'                                    }
+    - { port: '9001-9005', comment: 'Beacon Node discovery', protocol: 'udp'                                    }
+    - { port: '9201-9205', comment: 'Beacon Node Metrics',   ipset: 'metrics.hq',                iifname: 'wg0' }
+    - { port: '9301-9305', comment: 'Beacon Node REST API',  ipset: '{{ env }}.{{ stage }}',     iifname: 'wg0' }
 
 # Fleet layout can be found in: ansible/vars/layout/mainnet.yml

ansible/mainnet.yml

@@ -19,7 +19,7 @@
     beacon_node_netkey: '{{lookup("bitwarden", "nimbus/netkey", field=hostname)}}'
   roles:
     - { role: infra-role-swap-file,         tags: [ swap-file ] }
-    - { role: infra-role-open-ports,        tags: [ open-ports ] }    
+    - { role: infra-role-open-ports,        tags: [ open-ports ] }
     - { role: redirect-ports,               tags: [ redirect-ports ] }
     - { role: get-geth-api-urls,            tags: [ get-geth-api-urls, beacon-node ] }
   tasks:
@@ -35,7 +35,10 @@
 - name: Deploy mainnet linux beacon nodes
   become: true
   serial: '{{ serial|default(1) }}'
-  hosts: nimbus-mainnet-metal
+  hosts:
+    - nimbus-mainnet-geth
+    - nimbus-mainnet-erigon
+    - nimbus-mainnet-nel
   vars_files: layout/mainnet.yml
   vars:
     # FIXME: Use all Geth nodes available in the fleet until we resync.
@@ -46,8 +49,6 @@
     - { role: infra-role-open-ports,        tags: [ open-ports ] }
     - { role: infra-role-smart-metrics,     tags: [ smart-metrics ] }
     - { role: redirect-ports,               tags: [ redirect-ports ] }
-    - { role: infra-role-geth,              tags: [ infra-role-geth ] }
-    - { role: infra-role-geth-exporter,     tags: [ infra-role-geth-exporter ] }
     - { role: get-geth-api-urls,            tags: [ get-geth-api-urls, beacon-node ] }
     - { role: nimbus-era-files,             tags: [ nimbus-era-files ] }
   tasks:
@@ -59,3 +60,59 @@
       loop_control:
         loop_var: node
         index_var: idx
+
+- name: Deploy Mainnet Geth Nodes
+  become: true
+  serial: '{{ serial|default(1) }}'
+  hosts: nimbus-mainnet-geth
+  vars_files: layout/mainnet.yml
+  tasks:
+    - include_role:
+        name: infra-role-geth
+        apply:
+          tags: always
+      tags: [ geth ]
+      when:
+        - "'el' in node"
+        - node.el == "geth"
+      with_items: '{{ nodes_layout[inventory_hostname] }}'
+      loop_control: { loop_var: node, index_var: idx }
+
+    - include_role:
+        name: infra-role-geth-exporter
+        apply:
+          tags: always
+      tags: [ geth-exporter ]
+      when: node.el == "geth"
+      with_items: '{{ nodes_layout[inventory_hostname] }}'
+      loop_control: { loop_var: node, index_var: idx }
+
+- name: Deploy Mainnet Erigon Nodes
+  become: true
+  serial: '{{ serial|default(1) }}'
+  hosts: nimbus-mainnet-erigon
+  vars_files: layout/mainnet.yml
+  tasks:
+    - include_role:
+        name: infra-role-erigon
+        apply:
+          tags: always
+      tags: [ erigon ]
+      when: node.el == "erigon"
+      with_items: '{{ nodes_layout[inventory_hostname] }}'
+      loop_control: { loop_var: node, index_var: idx }
+
+- name: Deploy Mainnet Nimbus ETH1 Nodes
+  become: true
+  serial: '{{ serial|default(1) }}'
+  hosts: nimbus-mainnet-nel
+  vars_files: layout/mainnet.yml
+  tasks:
+    - include_role:
+        name: infra-role-nimbus-eth1
+        apply:
+          tags: always
+      tags: [ nimbus-eth1 ]
+      when: node.el == "nel"
+      with_items: '{{ nodes_layout[inventory_hostname] }}'
+      loop_control: { loop_var: node, index_var: idx }

ansible/requirements.yml

@@ -50,7 +50,7 @@
 
 - name: infra-role-nimbus-eth1
   src: [email protected]:status-im/infra-role-nimbus-eth1.git
-  version: a9e82b5193d4930962f6a62e79d3b431a4030823
+  version: cd87a47daea4b4751588f32eb1d1b964baa1776c
 
 - name: infra-role-nimbus-fluffy
   src: [email protected]:status-im/infra-role-nimbus-fluffy.git

ansible/vars/addresses/update.sh

@@ -16,5 +16,7 @@ function update_yaml() {
 for FILE in geth-*.holesky.yml;   do update_yaml "${FILE}"; done
 for FILE in erigon-*.holesky.yml; do update_yaml "${FILE}"; done
 for FILE in neth-*.holesky.yml;   do update_yaml "${FILE}"; done
-for FILE in linux-*.mainnet.yml;  do update_yaml "${FILE}"; done
+for FILE in nel-*.mainnet.yml;    do update_yaml "${FILE}"; done
+for FILE in erigon-*.mainnet.yml; do update_yaml "${FILE}"; done
+for FILE in geth-*.mainnet.yml;   do update_yaml "${FILE}"; done
 for FILE in linux-*.sepolia.yml;  do update_yaml "${FILE}"; done