From e84c60f88af443bf9f13643e988e01b291c8b83c Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 9 Aug 2022 15:55:04 +0200 Subject: [PATCH 1/6] Better management of jemalloc presence/absence Signed-off-by: Fabio Di Fabio --- besu/src/main/scripts/unixStartScript.txt | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/besu/src/main/scripts/unixStartScript.txt b/besu/src/main/scripts/unixStartScript.txt index 50585f07660..568b01be946 100644 --- a/besu/src/main/scripts/unixStartScript.txt +++ b/besu/src/main/scripts/unixStartScript.txt @@ -182,8 +182,19 @@ APP_ARGS=`save "\$@"` # Collect all arguments for the java command, following the shell quoting and substitution rules eval set -- \$DEFAULT_JVM_OPTS \$JAVA_OPTS \$${optsEnvironmentVar} <% if ( appNameSystemProperty ) { %>"\"-D${appNameSystemProperty}=\$APP_BASE_NAME\"" <% } %>-classpath "\"\$CLASSPATH\"" <% if ( mainClassName.startsWith('--module ') ) { %>--module-path "\"\$MODULE_PATH\"" <% } %>${mainClassName} "\$APP_ARGS" -# limit malloc to 2 arenas, and use jemalloc if available -export MALLOC_ARENA_MAX=2 -export LD_PRELOAD=libjemalloc.so +if [ "\$darwin" = "false" -a "\$msys" = "false" ]; then + # check if jemalloc is available + TEST_JEMALLOC=\$(LD_PRELOAD=libjemalloc.so sh -c true 2>&1) + + # if jemalloc is available the output is empty, otherwise the output has an error line + if [ -z "\$TEST_JEMALLOC" ]; then + echo "INFO: Enabling jemalloc." + export LD_PRELOAD=libjemalloc.so + else + echo "INFO: jemalloc not found, you could improve memory usage installing it." + # jemalloc not available, as fallback limit malloc to 2 arenas + export MALLOC_ARENA_MAX=2 + fi +fi exec "\$JAVACMD" "\$@" From 73a6a2cb176d893fc104e6066fc0ee9f8f11b1bc Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 9 Aug 2022 16:00:48 +0200 Subject: [PATCH 2/6] Add CHANGELOG Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + besu/src/main/scripts/unixStartScript.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98ae3516e07..c4988a7729c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Add experimental CLI option for `--Xp2p-peer-lower-bound` [#4200](https://github.com/hyperledger/besu/pull/4200) - Improve pending blocks retrieval mechanism [#4227](https://github.com/hyperledger/besu/pull/4227) - set mainnet terminal total difficulty [#4260](https://github.com/hyperledger/besu/pull/4260) +- Better management of jemalloc presence/absence in startup script [#4237](https://github.com/hyperledger/besu/pull/4237) ### Bug Fixes - Fixes off-by-one error for mainnet TTD fallback [#4223](https://github.com/hyperledger/besu/pull/4223) diff --git a/besu/src/main/scripts/unixStartScript.txt b/besu/src/main/scripts/unixStartScript.txt index 568b01be946..99f9c2be3b1 100644 --- a/besu/src/main/scripts/unixStartScript.txt +++ b/besu/src/main/scripts/unixStartScript.txt @@ -188,7 +188,6 @@ if [ "\$darwin" = "false" -a "\$msys" = "false" ]; then # if jemalloc is available the output is empty, otherwise the output has an error line if [ -z "\$TEST_JEMALLOC" ]; then - echo "INFO: Enabling jemalloc." export LD_PRELOAD=libjemalloc.so else echo "INFO: jemalloc not found, you could improve memory usage installing it." From 8d785166673aa227c9750c0457ebcaa31ceb306c Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Thu, 18 Aug 2022 11:26:57 +0200 Subject: [PATCH 3/6] Move log about jemalloc from startup script to Java Signed-off-by: Fabio Di Fabio --- .../org/hyperledger/besu/cli/BesuCommand.java | 13 +++++++++++++ besu/src/main/scripts/unixStartScript.txt | 3 ++- .../hyperledger/besu/cli/BesuCommandTest.java | 17 +++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index d1ec8d28b15..3383fc179ed 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -185,6 +185,7 @@ import org.hyperledger.besu.util.number.Fraction; import org.hyperledger.besu.util.number.Percentage; import org.hyperledger.besu.util.number.PositiveNumber; +import org.hyperledger.besu.util.platform.PlatformDetector; import java.io.File; import java.io.IOException; @@ -1391,6 +1392,7 @@ public void parse( handleUnstableOptions(); preparePlugins(); parse(resultHandler, exceptionHandler, args); + detectJemalloc(); } @Override @@ -1492,6 +1494,17 @@ private void registerConverters() { commandLine.registerConverter(MetricCategory.class, metricCategoryConverter); } + private void detectJemalloc() { + // jemalloc is only supported on Linux at the moment + if (PlatformDetector.getOSType().equals("linux")) { + Optional.ofNullable(environment.get("BESU_USING_JEMALLOC")) + .ifPresentOrElse( + present -> logger.info("Using jemalloc"), + () -> + logger.info("jemalloc not found, you could improve memory usage installing it")); + } + } + private void handleStableOptions() { commandLine.addMixin("Ethstats", ethstatsOptions); commandLine.addMixin("Private key file", nodePrivateKeyFileOption); diff --git a/besu/src/main/scripts/unixStartScript.txt b/besu/src/main/scripts/unixStartScript.txt index 99f9c2be3b1..3302b9c2fe1 100644 --- a/besu/src/main/scripts/unixStartScript.txt +++ b/besu/src/main/scripts/unixStartScript.txt @@ -182,6 +182,7 @@ APP_ARGS=`save "\$@"` # Collect all arguments for the java command, following the shell quoting and substitution rules eval set -- \$DEFAULT_JVM_OPTS \$JAVA_OPTS \$${optsEnvironmentVar} <% if ( appNameSystemProperty ) { %>"\"-D${appNameSystemProperty}=\$APP_BASE_NAME\"" <% } %>-classpath "\"\$CLASSPATH\"" <% if ( mainClassName.startsWith('--module ') ) { %>--module-path "\"\$MODULE_PATH\"" <% } %>${mainClassName} "\$APP_ARGS" +unset BESU_USING_JEMALLOC if [ "\$darwin" = "false" -a "\$msys" = "false" ]; then # check if jemalloc is available TEST_JEMALLOC=\$(LD_PRELOAD=libjemalloc.so sh -c true 2>&1) @@ -189,8 +190,8 @@ if [ "\$darwin" = "false" -a "\$msys" = "false" ]; then # if jemalloc is available the output is empty, otherwise the output has an error line if [ -z "\$TEST_JEMALLOC" ]; then export LD_PRELOAD=libjemalloc.so + export BESU_USING_JEMALLOC=true else - echo "INFO: jemalloc not found, you could improve memory usage installing it." # jemalloc not available, as fallback limit malloc to 2 arenas export MALLOC_ARENA_MAX=2 fi diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 1b64a1cdb65..cecd09adb64 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -17,6 +17,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.startsWith; import static org.hyperledger.besu.cli.config.NetworkName.CLASSIC; import static org.hyperledger.besu.cli.config.NetworkName.DEV; @@ -94,6 +95,7 @@ import org.hyperledger.besu.plugin.services.rpc.PluginRpcRequest; import org.hyperledger.besu.util.number.Fraction; import org.hyperledger.besu.util.number.Percentage; +import org.hyperledger.besu.util.platform.PlatformDetector; import java.io.File; import java.io.IOException; @@ -5318,4 +5320,19 @@ public void pkiBlockCreationFullConfig() throws Exception { assertThat(pkiKeyStoreConfig.getTrustStorePassword()).isEqualTo("foo"); assertThat(pkiKeyStoreConfig.getCrlFilePath()).hasValue(Path.of("/tmp/crl")); } + + @Test + public void logsUsingJemallocWhenEnvVarPresent() { + assumeThat(PlatformDetector.getOSType(), is("linux")); + setEnvironmentVariable("BESU_USING_JEMALLOC", "true"); + parseCommand(); + verify(mockLogger).info("Using jemalloc"); + } + + @Test + public void logsSuggestInstallingJemallocWhenEnvVarNotPresent() { + assumeThat(PlatformDetector.getOSType(), is("linux")); + parseCommand(); + verify(mockLogger).info("jemalloc not found, you could improve memory usage installing it"); + } } From 546d3de9b53298bfe3add58345c8cfc66623734d Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 19 Aug 2022 11:40:00 +0200 Subject: [PATCH 4/6] Wording change for the log message Co-authored-by: Adrian Sutton Signed-off-by: Fabio Di Fabio --- besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 3383fc179ed..f86d02be513 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1501,7 +1501,7 @@ private void detectJemalloc() { .ifPresentOrElse( present -> logger.info("Using jemalloc"), () -> - logger.info("jemalloc not found, you could improve memory usage installing it")); + logger.info("jemalloc library not found, memory usage may be reduced by installing it")); } } From 1e5f0534432b0803665c24dadf3f3ff530d4e0ab Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 19 Aug 2022 14:04:09 +0200 Subject: [PATCH 5/6] fix test and spotless Signed-off-by: Fabio Di Fabio --- besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java | 3 ++- .../test/java/org/hyperledger/besu/cli/BesuCommandTest.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index f86d02be513..9cb91613cb2 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1501,7 +1501,8 @@ private void detectJemalloc() { .ifPresentOrElse( present -> logger.info("Using jemalloc"), () -> - logger.info("jemalloc library not found, memory usage may be reduced by installing it")); + logger.info( + "jemalloc library not found, memory usage may be reduced by installing it")); } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index cecd09adb64..9878140ccd4 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -5333,6 +5333,7 @@ public void logsUsingJemallocWhenEnvVarPresent() { public void logsSuggestInstallingJemallocWhenEnvVarNotPresent() { assumeThat(PlatformDetector.getOSType(), is("linux")); parseCommand(); - verify(mockLogger).info("jemalloc not found, you could improve memory usage installing it"); + verify(mockLogger) + .info("jemalloc library not found, memory usage may be reduced by installing it"); } } From 887dcf0d9ecc921fdcda674dd13594f475c0c69c Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 23 Aug 2022 12:02:49 +0200 Subject: [PATCH 6/6] Update CHANGELOG Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff7ff975101..0c7d298ee97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Additions and Improvements - Upgrade besu-native to 0.6.0 and use Blake2bf native implementation if available by default [#4264](https://github.com/hyperledger/besu/pull/4264) +- Better management of jemalloc presence/absence in startup script [#4237](https://github.com/hyperledger/besu/pull/4237) ### Bug Fixes @@ -15,7 +16,6 @@ - Add experimental CLI option for `--Xp2p-peer-lower-bound` [#4200](https://github.com/hyperledger/besu/pull/4200) - Improve pending blocks retrieval mechanism [#4227](https://github.com/hyperledger/besu/pull/4227) - set mainnet terminal total difficulty [#4260](https://github.com/hyperledger/besu/pull/4260) -- Better management of jemalloc presence/absence in startup script [#4237](https://github.com/hyperledger/besu/pull/4237) ### Bug Fixes - Fixes off-by-one error for mainnet TTD fallback [#4223](https://github.com/hyperledger/besu/pull/4223) @@ -35,7 +35,7 @@ ### Additions and Improvements - Deprecation warning for Ropsten, Rinkeby, Kiln [#4173](https://github.com/hyperledger/besu/pull/4173) -### Bug Fixes +### Bug Fixes - Fixes previous known issue [#3890](https://github.com/hyperledger/besu/issues/3890)from RC3 requiring a restart post-merge to continue correct transaction handling. - Stop producing stack traces when a get headers response only contains the range start header [#4189](https://github.com/hyperledger/besu/pull/4189) @@ -54,8 +54,8 @@ ### Additions and Improvements - Engine API: Change expiration time for JWT tokens to 60s [#4168](https://github.com/hyperledger/besu/pull/4168) - Sepolia mergeNetSplit block [#4158](https://github.com/hyperledger/besu/pull/4158) -- Goerli TTD [#4160](https://github.com/hyperledger/besu/pull/4160) -- Several logging improvements +- Goerli TTD [#4160](https://github.com/hyperledger/besu/pull/4160) +- Several logging improvements ### Bug Fixes - Allow to set any value for baseFeePerGas in the genesis file [#4177](https://github.com/hyperledger/besu/pull/4177)