From 954f96f2d8561722d45e8bafdd0b6d8111f4d8ff Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 30 Mar 2021 16:02:37 +1100 Subject: [PATCH] Squashed commit of #2222 commit 13adea78354beb4ff406bd8d5748b560feb37042 Merge: 345802fff a764c3b24 Author: Paul Hauner Date: Wed Feb 24 16:21:22 2021 +1100 Merge branch 'unstable' into jemalloc-paul commit 345802fff7b6a0da4088531c09ac20a99af0769f Author: Paul Hauner Date: Wed Feb 24 15:58:31 2021 +1100 Jemalloc by default commit 88e54abbff18d74949b4310cee6f6b9062df8b99 Author: Paul Hauner Date: Wed Feb 24 15:41:35 2021 +1100 Fix config commit 6680b6d8a962fbed6ba9e0a09d11d6920cb708eb Merge: 42eb539fa 46920a84e Author: Paul Hauner Date: Wed Feb 24 14:21:45 2021 +1100 Merge branch 'unstable' into jemalloc-sea commit 42eb539fa2d8c2fc5ecbc1c3e6a866a0079ae458 Author: Paul Hauner Date: Tue Feb 16 12:12:34 2021 +1100 Set jemalloc as default, configure arenas commit 4331ef865408fb9f9c2606d32aa5c637e336f646 Merge: d4010e345 4a357c994 Author: Paul Hauner Date: Tue Feb 16 10:00:47 2021 +1100 Merge branch 'unstable' into jemalloc-sea commit d4010e34562ad953e63339ec11056f97494fb24c Author: Paul Hauner Date: Tue Feb 16 10:00:37 2021 +1100 Make uses jemmaloc commit 5e968df9bce24730baf1d197c8a6cb7308b21ac6 Merge: ff14ee057 e2ff9c66a Author: realbigsean Date: Fri Feb 12 10:36:49 2021 -0500 Merge branch 'unstable' of https://github.com/sigp/lighthouse into jemalloc  Conflicts:  Cargo.lock  beacon_node/http_metrics/Cargo.toml  beacon_node/network/src/attestation_service/tests/mod.rs commit ff14ee0576818489a7f004b27b4e02071d855370 Author: realbigsean Date: Fri Feb 12 10:35:18 2021 -0500 add newlines commit ce1c4e1d7bd69fa12756bcf7288548912d1dcf3d Author: realbigsean Date: Fri Feb 12 09:54:19 2021 -0500 move jemalloc to a feature commit a0d71191aa67e76ce55bc0e0490c02cfbfc3cf48 Author: realbigsean Date: Thu Feb 11 19:12:13 2021 -0500 cargo lock commit f205c1894034f9ca452dd0115c55f9283864b777 Author: realbigsean Date: Thu Feb 11 18:53:43 2021 -0500 delete unused dep commit fd171928382f8b2b4792ed8cfa9ad7bc3d0fdfc1 Author: realbigsean Date: Thu Feb 11 18:51:25 2021 -0500 delete background threads commit ae8b61a2af8a2bc1b8dcfbcdd533b82cfee7ee59 Author: realbigsean Date: Thu Feb 11 18:50:55 2021 -0500 fix error handling and remove background threads commit 9e7a6b7b4d34dd49340d69f3c61ab5533ba7ed06 Author: realbigsean Date: Thu Feb 11 11:48:25 2021 -0500 semicolon commit 3408ee12f1f8fd4d5722a88050ed4f9b13ec89c9 Author: realbigsean Date: Thu Feb 11 11:46:40 2021 -0500 update to associated function commit 074ce1fb70edfab766b66db66873a4957c8194c4 Author: realbigsean Date: Thu Feb 11 11:43:25 2021 -0500 enable background threads commit 8805ef9b202a90b03b9aa5b3ee42583e106310a3 Author: realbigsean Date: Thu Feb 11 10:34:47 2021 -0500 fix arenas commit 6f17971956460a62aa74f46186c3df15f9bc8d15 Author: realbigsean Date: Thu Feb 11 10:25:09 2021 -0500 import arenas commit 034b677f6147ca0addfc1ca0caf2c69e2170e197 Author: realbigsean Date: Thu Feb 11 10:23:12 2021 -0500 add more jemalloc stats commit 632bff2d96799eed01f0e16777b7977b19a49061 Author: realbigsean Date: Wed Feb 10 19:11:30 2021 -0500 test add jemalloc metrics commit abf5cac65b2464fccd5c11547c4f0bdd656e5389 Author: realbigsean Date: Wed Feb 10 19:03:08 2021 -0500 test add jemalloc metrics commit 1d1fc6ae11dd53a6c21fd3cace9e2db2c2425f98 Author: realbigsean Date: Wed Feb 10 18:48:48 2021 -0500 add jemalloc deps commit 3c7920b592500d762583b6fdc8daea0679fdb9d7 Author: realbigsean Date: Wed Feb 10 18:34:18 2021 -0500 test add jemalloc metrics commit db6fda6a4fbb62c48ed9af9d3681f0675f659e34 Author: realbigsean Date: Wed Feb 10 18:14:22 2021 -0500 test add jemalloc --- beacon_node/Cargo.toml | 2 ++ beacon_node/client/Cargo.toml | 4 +++ beacon_node/http_metrics/Cargo.toml | 4 +++ common/warp_utils/Cargo.toml | 5 +++ common/warp_utils/src/metrics.rs | 55 +++++++++++++++++++++++++++++ lighthouse/Cargo.toml | 5 +++ lighthouse/src/main.rs | 21 +++++++++++ 7 files changed, 96 insertions(+) diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 2495bbcaaca..52d6363015a 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -13,6 +13,8 @@ node_test_rig = { path = "../testing/node_test_rig" } [features] write_ssz_files = ["beacon_chain/write_ssz_files"] # Writes debugging .ssz files to /tmp during block processing. +# Use the system allocator instead of jemalloc. This replicates default Rust behaviour. +sysalloc = ["client/sysalloc"] [dependencies] eth2_config = { path = "../common/eth2_config" } diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 28dea4055d6..1c40ca67d74 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -44,3 +44,7 @@ http_api = { path = "../http_api" } http_metrics = { path = "../http_metrics" } slasher = { path = "../../slasher" } slasher_service = { path = "../../slasher/service" } + +[features] +# Use the system allocator instead of jemalloc. This replicates default Rust behaviour. +sysalloc = ["http_metrics/sysalloc"] diff --git a/beacon_node/http_metrics/Cargo.toml b/beacon_node/http_metrics/Cargo.toml index 78394874648..77c61a4fdeb 100644 --- a/beacon_node/http_metrics/Cargo.toml +++ b/beacon_node/http_metrics/Cargo.toml @@ -26,3 +26,7 @@ tokio = { version = "1.1.0", features = ["sync"] } reqwest = { version = "0.11.0", features = ["json"] } environment = { path = "../../lighthouse/environment" } types = { path = "../../consensus/types" } + +[features] +# Use the system allocator instead of jemalloc. This replicates default Rust behaviour. +sysalloc = ["warp_utils/sysalloc"] diff --git a/common/warp_utils/Cargo.toml b/common/warp_utils/Cargo.toml index 7b2ab637398..e5729db382c 100644 --- a/common/warp_utils/Cargo.toml +++ b/common/warp_utils/Cargo.toml @@ -18,3 +18,8 @@ tokio = { version = "1.1.0", features = ["sync"] } headers = "0.3.2" lighthouse_metrics = { path = "../lighthouse_metrics" } lazy_static = "1.4.0" +jemalloc-ctl = { version = "0.3.3" } + +[features] +# Use the system allocator instead of jemalloc. This replicates default Rust behaviour. +sysalloc = [] diff --git a/common/warp_utils/src/metrics.rs b/common/warp_utils/src/metrics.rs index dc42aa6b357..0ae2f32abf1 100644 --- a/common/warp_utils/src/metrics.rs +++ b/common/warp_utils/src/metrics.rs @@ -1,6 +1,9 @@ use eth2::lighthouse::Health; use lighthouse_metrics::*; +#[cfg(not(feature = "sysalloc"))] +use jemalloc_ctl::{arenas, epoch, stats}; + lazy_static::lazy_static! { pub static ref PROCESS_NUM_THREADS: Result = try_create_int_gauge( "process_num_threads", @@ -34,6 +37,24 @@ lazy_static::lazy_static! { try_create_float_gauge("system_loadavg_5", "Loadavg over 5 minutes"); pub static ref SYSTEM_LOADAVG_15: Result = try_create_float_gauge("system_loadavg_15", "Loadavg over 15 minutes"); + + /* + * jemalloc + */ + pub static ref JEMALLOC_RESIDENT: Result = + try_create_int_gauge("jemalloc_resident", "Total number of bytes in physically resident data pages mapped by the allocator."); + pub static ref JEMALLOC_ALLOCATED: Result = + try_create_int_gauge("jemalloc_allocated", "Total number of bytes allocated by the application."); + pub static ref JEMALLOC_MAPPED: Result = + try_create_int_gauge("jemalloc_mapped", "Total number of bytes in active extents mapped by the allocator."); + pub static ref JEMALLOC_METADATA: Result = + try_create_int_gauge("jemalloc_metadata", "Total number of bytes dedicated to jemalloc metadata."); + pub static ref JEMALLOC_RETAINED: Result = + try_create_int_gauge("jemalloc_retained", "Total number of bytes in virtual memory mappings that were retained rather than being returned to the operating system."); + pub static ref JEMALLOC_ACTIVE: Result = + try_create_int_gauge("jemalloc_active", "Total number of bytes in active pages allocated by the application."); + pub static ref JEMALLOC_ARENAS: Result = + try_create_int_gauge("jemalloc_arenas", "Current limit on the number of arenas."); } pub fn scrape_health_metrics() { @@ -58,4 +79,38 @@ pub fn scrape_health_metrics() { set_float_gauge(&SYSTEM_LOADAVG_5, health.sys_loadavg_5); set_float_gauge(&SYSTEM_LOADAVG_15, health.sys_loadavg_15); } + + scrape_jemalloc_metrics(); +} + +#[cfg(not(feature = "sysalloc"))] +pub fn scrape_jemalloc_metrics() { + if epoch::advance().is_ok() { + if let Ok(allocated) = stats::allocated::read() { + set_gauge(&JEMALLOC_ALLOCATED, allocated as i64); + } + if let Ok(resident) = stats::resident::read() { + set_gauge(&JEMALLOC_RESIDENT, resident as i64); + } + if let Ok(mapped) = stats::mapped::read() { + set_gauge(&JEMALLOC_MAPPED, mapped as i64); + } + if let Ok(metadata) = stats::metadata::read() { + set_gauge(&JEMALLOC_METADATA, metadata as i64); + } + if let Ok(retained) = stats::retained::read() { + set_gauge(&JEMALLOC_RETAINED, retained as i64); + } + if let Ok(active) = stats::active::read() { + set_gauge(&JEMALLOC_ACTIVE, active as i64); + } + if let Ok(narenas) = arenas::narenas::read() { + set_gauge(&JEMALLOC_ARENAS, narenas as i64); + } + } +} + +#[cfg(feature = "sysalloc")] +pub fn scrape_jemalloc_metrics() { + // NO OP } diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index a96112e392d..68c34e90efb 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -17,6 +17,8 @@ milagro = ["bls/milagro"] spec-minimal = [] # Support spec v0.12 (used by Medalla testnet). spec-v12 = [] +# Use the system allocator instead of jemalloc. This replicates default Rust behaviour. +sysalloc = ["beacon_node/sysalloc"] [dependencies] beacon_node = { "path" = "../beacon_node" } @@ -43,6 +45,9 @@ account_utils = { path = "../common/account_utils" } remote_signer = { "path" = "../remote_signer" } lighthouse_metrics = { path = "../common/lighthouse_metrics" } lazy_static = "1.4.0" +jemallocator = { version = "0.3.2" } +jemalloc-sys = { version = "0.3.2" } +libc = { version = "0.2.86" } [dev-dependencies] tempfile = "3.1.0" diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index acbc80a8468..fedc6cf8f37 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -12,6 +12,27 @@ use std::process::exit; use types::{EthSpec, EthSpecId}; use validator_client::ProductionValidatorClient; +/// Global allocator +#[cfg(not(feature = "sysalloc"))] +#[global_allocator] +pub static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; + +#[cfg(not(feature = "sysalloc"))] +union U { + x: &'static u8, + y: &'static libc::c_char, +} + +#[cfg(not(feature = "sysalloc"))] +#[allow(non_upper_case_globals)] +#[export_name = "_rjem_malloc_conf"] +pub static malloc_conf: Option<&'static libc::c_char> = Some(unsafe { + U { + x: &b"narenas:1\0"[0], + } + .y +}); + pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml"; fn bls_library_name() -> &'static str {