mainnet: change layout

TODO: - #211 - #212
status-im · Oct 23, 2024 · a0dcb9c · a0dcb9c
1 parent acb3297
commit a0dcb9c
Show file tree

Hide file tree

Showing 21 changed files with 308 additions and 162 deletions.
diff --git a/README.md b/README.md
@@ -8,8 +8,8 @@ These are [Beacon API](https://ethereum.github.io/beacon-APIs/) endpoints intend
 
 | Endpoint                                        | Host                                 |
 |-------------------------------------------------|--------------------------------------|
-| http://unstable.mainnet.beacon-api.nimbus.team/ | `linux-01.ih-eu-mda1.nimbus.mainnet` |
-| http://testing.mainnet.beacon-api.nimbus.team/  | `linux-02.ih-eu-mda1.nimbus.mainnet` |
+| http://unstable.mainnet.beacon-api.nimbus.team/ | `geth-01.ih-eu-mda1.nimbus.mainnet`  |
+| http://testing.mainnet.beacon-api.nimbus.team/  | `geth-02.ih-eu-mda1.nimbus.mainnet`  |
 | http://unstable.sepolia.beacon-api.nimbus.team/ | `linux-01.ih-eu-mda1.nimbus.sepolia` |
 | http://unstable.holesky.beacon-api.nimbus.team/ | `geth-01.ih-eu-mda1.nimbus.holesky`  |
 | http://testing.holesky.beacon-api.nimbus.team/  | `geth-02.ih-eu-mda1.nimbus.holesky`  |
@@ -18,12 +18,12 @@ These nodes have no validators attached.
 
 There are also archives of ERA files:
 
-| Endpoint                          | Host                                 |
-|-----------------------------------|--------------------------------------|
-| https://mainnet.era.nimbus.team/  | `linux-03.ih-eu-mda1.nimbus.mainnet` |
-| https://sepolia.era.nimbus.team/  | `linux-01.ih-eu-mda1.nimbus.sepolia` |
-| https://sepolia.era1.nimbus.team/ | `linux-01.ih-eu-mda1.nimbus.sepolia` |
-| https://holesky.era.nimbus.team/  | `geth-01.ih-eu-mda1.nimbus.holesky`  |
+| Endpoint                          | Host                                  |
+|-----------------------------------|---------------------------------------|
+| https://mainnet.era.nimbus.team/  | `erigon-01.ih-eu-mda1.nimbus.mainnet` |
+| https://sepolia.era.nimbus.team/  | `linux-01.ih-eu-mda1.nimbus.sepolia`  |
+| https://sepolia.era1.nimbus.team/ | `linux-01.ih-eu-mda1.nimbus.sepolia`  |
+| https://holesky.era.nimbus.team/  | `geth-01.ih-eu-mda1.nimbus.holesky`   |
 
 # Dashboards
 

diff --git a/ansible/era.yml b/ansible/era.yml
@@ -12,7 +12,7 @@
 - name: Configure ERA files hosting
   become: true
   hosts:
-    - linux-03.ih-eu-mda1.nimbus.mainnet
+    - erigon-01.ih-eu-mda1.nimbus.mainnet
     - linux-01.ih-eu-mda1.nimbus.sepolia
     - geth-01.ih-eu-mda1.nimbus.holesky
   roles:

diff --git a/ansible/group_vars/nimbus.mainnet.yml b/ansible/group_vars/nimbus.mainnet.yml
@@ -6,82 +6,128 @@ bootstrap__extra_volume_bind_path: null
 # SWAP
 swap_file_size_mb: 2048
 
-# Go-Ethereum
-geth_service_name: 'geth-{{ geth_network_name }}'
+# Ports used for all 3 EL nodes as they are mutually exclusive.
+exec_layer_p2p_port:     '{{ 30303 + 3*idx|int + 1 }}'
+exec_layer_rpc_port:     '{{ 8545  +   idx|int + 1 }}'
+exec_layer_authrpc_port: '{{ 8551  +   idx|int + 1 }}'
+exec_layer_metrics_port: '{{ 6060  +   idx|int + 1 }}'
+
+# Go-Ethereum ------------------------------------------------------------------
+
+geth_service_name: 'geth-{{ geth_network_name }}-{{ node.branch | mandatory }}'
 geth_service_path: '/docker/{{ geth_service_name }}'
 geth_network_name: 'mainnet'
 geth_cont_name: '{{ geth_service_name }}-node'
 geth_cont_vol: '{{ geth_service_path }}/node'
 geth_sync_mode: 'snap'
 geth_log_level_name: 'info'
+# Geth auth & JWT token
 geth_account_pass: '{{lookup("bitwarden", "nimbus/geth", field="password")}}'
-geth_authrpc_jwtsecret: '{{lookup("bitwarden", "nimbus/jwt-token")}}'
-# Memory settings¬
+geth_authrpc_jwtsecret: '{{ beacon_node_exec_layer_jwt_secret }}'
+# Memory settings
 geth_cont_mem_ratio: 0.15
 geth_cache_size: '{{ (ansible_memtotal_mb * 0.05|float) | int }}'
 # Ports
-geth_port: 30303
-geth_rpc_addr: '0.0.0.0'
-geth_rpc_port: 8545
-geth_authrpc_addr: '127.0.0.1'
-geth_authrpc_port: 8551
-# Geth metrics¬
+geth_rpc_addr:        '0.0.0.0'
+geth_authrpc_addr:    '127.0.0.1'
+geth_port:            '{{ exec_layer_p2p_port }}'
+geth_rpc_port:        '{{ exec_layer_rpc_port }}'
+geth_authrpc_port:    '{{ exec_layer_authrpc_port }}'
+geth_metrics_port:    '{{ exec_layer_metrics_port }}'
+# Geth metrics
 geth_expo_service_name: '{{ geth_service_name }}'
 geth_expo_source_cont_name: '{{ geth_cont_name }}'
 geth_expo_source_data_path: '{{ geth_cont_vol }}/data'
-geth_expo_cont_port: 9400
+geth_expo_cont_port:  '{{ 9400 + (idx|int) + 1 }}'
+
+# Erigon -----------------------------------------------------------------------
+
+erigon_network_name: 'holesky'
+erigon_service_name: 'erigon-{{ geth_network_name }}-{{ node.branch | mandatory }}'
+erigon_prune: 'hrtc' # TODO: Possibly wrong, verify.
+erigon_log_level: 'info'
+erigon_cont_mem_ratio: 0.15
+erigon_max_peers: 20
+erigon_miner_enabled: true
+# Ports
+erigon_metrics_enabled: true
+erigon_rpc_addr: '0.0.0.0'
+erigon_authrpc_addr: '0.0.0.0'
+erigon_port:              '{{ exec_layer_p2p_port }}'
+erigon_p2p_allowed_ports: ['{{ erigon_port }}', '{{ erigon_port|int + 1 }}']
+erigon_rpc_port:          '{{ exec_layer_rpc_port }}'
+erigon_metrics_port:      '{{ exec_layer_metrics_port }}'
+erigon_authrpc_port:      '{{ exec_layer_authrpc_port }}'
+erigon_authrpc_jwtsecret: '{{ beacon_node_exec_layer_jwt_secret }}'
+
+# Nimbus ETH1 ------------------------------------------------------------------
+
+nimbus_eth1_service_name: 'nel-{{ geth_network_name }}-{{ node.branch | mandatory }}'
+nimbus_eth1_repo_branch: 'master'
+nimbus_eth1_network: 'mainnet'
+nimbus_eth1_max_peers: 160
+nimbus_eth1_log_level: 'DEBUG'
+nimbus_eth1_service_path: '/docker/{{ nimbus_eth1_service_name }}'
+nimbus_eth1_engine_http_enabled: true
+nimbus_eth1_engine_ws_enabled: true
+nimbus_eth1_era_dir: '{{ nimbus_era_files_timer_path }}'
+nimbus_eth1_era1_dir: '/docker/era1'
+# Ports
+nimbus_eth1_listening_port: '{{ exec_layer_p2p_port }}'
+nimbus_eth1_discovery_port: '{{ exec_layer_p2p_port }}'
+nimbus_eth1_http_port:      '{{ exec_layer_rpc_port }}'
+nimbus_eth1_metrics_port:   '{{ exec_layer_metrics_port }}'
+nimbus_eth1_metrics_address: '0.0.0.0'
+# API secert
+nimbus_eth1_jwt_secret: '{{ beacon_node_exec_layer_jwt_secret }}'
+
+# Beacon Nodes -----------------------------------------------------------------
 
-# Nimbus Beacon Node
 beacon_node_network: 'mainnet'
+beacon_node_service_name: 'beacon-node-{{ beacon_node_network }}-{{ node.branch | mandatory }}'
+beacon_node_service_path: '/data/{{ beacon_node_service_name }}'
+beacon_node_era_dir_path: '{{ nimbus_era_files_timer_path }}'
 beacon_node_repo_branch: '{{ node_name_to_branch_map.get(node.branch, node.branch) }}'
+# Tuning
+beacon_node_history_retention: '{{ "archive" if node.get("public_api") else node.get("history", "prune") }}'
+beacon_node_extra_flags: '{{ node.extra_flags if node.extra_flags is defined else {} }}'
+beacon_node_cores_per_node: '{{ (ansible_processor_vcpus / nodes_layout[inventory_hostname]|length) | round(0, "ceil") | int }}'
+beacon_node_threads: '{{ (node.branch == "libp2p") | ternary(1, beacon_node_cores_per_node) }}'
 # We map short names to branches to avoid too long service names.
 node_name_to_branch_map:
   libp2p: 'nim-libp2p-auto-bump-unstable'
-# TODO Drop this once all nodes have an number.
-service_number: '{{ node.get("num", False) | ternary(("-%02d"|format(node.get("num", 0))), "") }}'
-beacon_node_service_name: 'beacon-node-{{ beacon_node_network }}-{{ node.branch | mandatory }}{{ service_number }}'
-beacon_node_service_path: '/data/{{ beacon_node_service_name }}'
-beacon_node_era_dir_path: '{{ nimbus_era_files_timer_path }}'
-# Ports
-beacon_node_discovery_port: '{{ 9000 + idx }}'
-beacon_node_listening_port: '{{ 9000 + idx }}'
-beacon_node_metrics_port:   '{{ 9200 + idx }}'
-beacon_node_rest_port:      '{{ beacon_node_rest_port_base + idx }}'
-beacon_node_rest_port_base: 9300
-beacon_node_rest_address:   '0.0.0.0'
-beacon_node_max_peers: '{{ node.get("max_peers", 320) }}'
-# Firewall
-beacon_node_firewall_libp2p_open: '{{ node.get("open_libp2p_ports", true) }}'
 # Builds
 beacon_node_update_build_targets: ['nimbus_beacon_node', 'ncli_db']
 beacon_node_update_build: '{{ beacon_node_repo_branch != "stable" }}'
 beacon_node_update_frequency: '*-*-* {{ "%02d" | format(idx * 2) }}:00:00'
-# Tuning
-beacon_node_cores_per_node: '{{ (ansible_processor_vcpus / nodes_layout[inventory_hostname]|length) | round(0, "ceil") | int }}'
-beacon_node_threads: '{{ (node.branch == "libp2p") | ternary(1, beacon_node_cores_per_node) }}'
-# FIXME: Temporary test to debug REST API timeout issues.
-# https://github.com/status-im/nimbus-eth2/issues/5838
-#beacon_node_rest_allow_origin: '{{ "*" if node.get("public_api") else "" }}'
-beacon_node_history_retention: '{{ "archive" if node.get("public_api") else node.get("history", "prune") }}'
 # Monitoring
 beacon_node_validator_monitor_auto: true
 beacon_node_validator_monitor_details: '{{ node.public_api is not defined or not node.public_api }}'
-# Mainnet validators run on a separate fleet.
-beacon_node_dist_validators_enabled: false
-# Excellent stress test and good service to the community.
-beacon_node_subscribe_all: true
-beacon_node_extra_flags: '{{ node.extra_flags if node.extra_flags is defined else {} }}'
+# Execution layer Enginer API
+beacon_node_exec_layer_jwt_secret: '{{lookup("bitwarden", "nimbus/jwt-token")}}'
+beacon_node_exec_layer_urls_local: ['http://localhost:{{ exec_layer_authrpc_port }}']
+beacon_node_exec_layer_urls: '{{ beacon_node_exec_layer_urls_local if node.get("el") else [] }}'
 # Light client data
 beacon_node_light_client_data_enabled: '{{ (node.public_api is defined and node.public_api) }}'
 beacon_node_light_client_data_serve: true
 beacon_node_light_client_data_import_mode: 'full'
-# Execution layer Enginer API
-beacon_node_exec_layer_jwt_secret: '{{ geth_authrpc_jwtsecret }}'
-beacon_node_exec_layer_urls_local_geth: ['http://localhost:{{ geth_authrpc_port|string }}']
-beacon_node_exec_layer_urls_all: '{{ beacon_node_exec_layer_urls_discovered_geth + beacon_node_exec_layer_urls_local_geth }}'
-beacon_node_exec_layer_urls: |-
-  {{ beacon_node_exec_layer_urls_all if inventory_hostname_short == "linux-01"
-    else (idx % 2 == 0) | ternary(beacon_node_exec_layer_urls_all, []) }}
+# Mainnet validators run on a separate fleet.
+beacon_node_dist_validators_enabled: false
+# Ports
+beacon_node_rest_port_base: 9300
+beacon_node_rest_port:      '{{ beacon_node_rest_port_base + idx|int + 1 }}'
+beacon_node_discovery_port: '{{ 9000 + idx|int + 1 }}'
+beacon_node_listening_port: '{{ 9000 + idx|int + 1 }}'
+beacon_node_metrics_port:   '{{ 9200 + idx|int + 1 }}'
+beacon_node_rest_address:   '0.0.0.0'
+beacon_node_max_peers: '{{ node.get("max_peers", 320) }}'
+# Firewall
+beacon_node_firewall_libp2p_open: '{{ node.get("open_libp2p_ports", true) }}'
+# Excellent stress test and good service to the community.
+beacon_node_subscribe_all: true
+# FIXME: Temporary test to debug REST API timeout issues.
+# https://github.com/status-im/nimbus-eth2/issues/5838
+#beacon_node_rest_allow_origin: '{{ "*" if node.get("public_api") else "" }}'
 
 # Reduce Consul alerts sensitivity
 beacon_node_consul_check_disabled: '{{ node.get("public_api", false) }}'
@@ -105,29 +151,31 @@ nimbus_era_files_timer_enabled: '{{ (nodes_layout[inventory_hostname]|length) >
 nimbus_era_files_timer_path: '/data/era'
 nimbus_era_files_network: '{{ beacon_node_network }}'
 # FIXME: Not pretty, since hardcoded, but the simplest way to do it right now.
-nimbus_era_files_db_path: '/data/beacon-node-{{ beacon_node_network }}-stable-01/data/db'
-nimbus_era_files_nclidb_path: '/data/beacon-node-{{ beacon_node_network }}-stable-01/repo/build/ncli_db'
+nimbus_era_files_db_path: '/data/beacon-node-{{ beacon_node_network }}-stable/data/db'
+nimbus_era_files_nclidb_path: '/data/beacon-node-{{ beacon_node_network }}-stable/repo/build/ncli_db'
 
 # Migrated to NFTables from IPTables.
 # https://github.com/status-im/infra-misc/issues/301
 bootstrap__firewall_nftables: true
 
 # Open Ports
+host_el_type: '{{ ansible_hostname|split("-")|first }}'
 open_ports_list:
   nginx:
     - { port: '443', comment: 'Nginx' }
   smart-metrics:
     - { port: '{{ smart_metrics_listen_port }}', comment: 'SMART Metrics', ipset: 'metrics.hq', iifname: 'wg0' }
-  geth-node:
-    - { port: '{{ geth_port }}',           comment: 'Go-Ethereum libp2p',    protocol: 'tcp'                                }
-    - { port: '{{ geth_port }}',           comment: 'Go-Ethereum discovery', protocol: 'udp'                                }
-    - { port: '{{ geth_expo_cont_port }}', comment: 'Go-Ethereum Exporter',  ipset: 'metrics.hq',            iifname: 'wg0' }
-    - { port: '{{ geth_authrpc_port }}',   comment: 'Go-Ethereum AuthRPC',   ipset: '{{ env }}.{{ stage }}', iifname: 'wg0' }
-    - { port: '{{ geth_rpc_port }}',       comment: 'RPC for Portal Bridge', ipset: 'nimbus.fluffy',         iifname: 'wg0' }
+  exec-node:
+    - { port: '30304-30315', comment: '{{ host_el_type }} LibP2P',    protocol: 'tcp'                     }
+    - { port: '30304-30315', comment: '{{ host_el_type }} Discovery', protocol: 'udp'                     }
+    - { port: '6061-6064',   comment: '{{ host_el_type }} Metrics',   ipset: 'metrics.hq', iifname: 'wg0' }
+    - { port: '9401-9404',   comment: '{{ host_el_type }} Exporter',  ipset: 'metrics.hq', iifname: 'wg0' }
+    - { port: '8552-8555',   comment: 'Go-Ethereum AuthRPC',   ipset: '{{ env }}.{{ stage }}', iifname: 'wg0' }
+    - { port: '8446-8449',   comment: 'RPC for Portal Bridge', ipset: 'nimbus.fluffy',         iifname: 'wg0' }
   beacon-node:
-    - { port: '9000-9005', comment: 'Beacon Node libp2p',    protocol: 'tcp'                                    }
-    - { port: '9000-9005', comment: 'Beacon Node discovery', protocol: 'udp'                                    }
-    - { port: '9200-9205', comment: 'Beacon Node Metrics',   ipset: 'metrics.hq',                iifname: 'wg0' }
-    - { port: '9300-9305', comment: 'Beacon Node REST API',  ipset: '{{ env }}.{{ stage }}',     iifname: 'wg0' }
+    - { port: '9001-9004', comment: 'Beacon Node libp2p',    protocol: 'tcp'                                    }
+    - { port: '9001-9004', comment: 'Beacon Node discovery', protocol: 'udp'                                    }
+    - { port: '9201-9204', comment: 'Beacon Node Metrics',   ipset: 'metrics.hq',                iifname: 'wg0' }
+    - { port: '9301-9304', comment: 'Beacon Node REST API',  ipset: '{{ env }}.{{ stage }}',     iifname: 'wg0' }
 
 # Fleet layout can be found in: ansible/vars/layout/mainnet.yml
diff --git a/...rs/linux-03.ih-eu-mda1.nimbus.mainnet.yml → ...s/erigon-01.ih-eu-mda1.nimbus.mainnet.yml b/...rs/linux-03.ih-eu-mda1.nimbus.mainnet.yml → ...s/erigon-01.ih-eu-mda1.nimbus.mainnet.yml
diff --git a/...rs/linux-01.ih-eu-mda1.nimbus.mainnet.yml → ...ars/geth-01.ih-eu-mda1.nimbus.mainnet.yml b/...rs/linux-01.ih-eu-mda1.nimbus.mainnet.yml → ...ars/geth-01.ih-eu-mda1.nimbus.mainnet.yml
@@ -15,4 +15,4 @@ redirect_ports:
   public-beacon-api:
     src: 80
     dst: '{{ test_api_beacon_node_rest_port }}'
-    comment: 'Public Beacon API ({{ test_api_beacon_node_branch }}-0{{ test_api_beacon_node_rest_index|int - 1 }})'
+    comment: 'Public Beacon API ({{ test_api_beacon_node_branch }})'
diff --git a/...rs/linux-02.ih-eu-mda1.nimbus.mainnet.yml → ...ars/geth-02.ih-eu-mda1.nimbus.mainnet.yml b/...rs/linux-02.ih-eu-mda1.nimbus.mainnet.yml → ...ars/geth-02.ih-eu-mda1.nimbus.mainnet.yml
@@ -16,4 +16,4 @@ redirect_ports:
   public-beacon-api:
     src: 80
     dst: '{{ test_api_beacon_node_rest_port }}'
-    comment: 'Public Beacon API ({{ test_api_beacon_node_branch }}-0{{ test_api_beacon_node_rest_index|int - 1 }})'
+    comment: 'Public Beacon API ({{ test_api_beacon_node_branch }})'