From ae60d90e5f316c29363007a3a978c883ecd8d388 Mon Sep 17 00:00:00 2001 From: Danny Thomas Date: Thu, 29 Aug 2024 03:12:57 +0000 Subject: [PATCH 1/8] Add Java Flight Recorder support --- spectator-ext-jvm/build.gradle | 50 +++++- .../spectator/jvm/JavaFlightRecorder.java | 53 ++++++ .../spectator/jvm/JavaFlightRecorder.java | 151 ++++++++++++++++++ .../JavaFlightRecorderUnsupportedTest.java | 36 +++++ .../spectator/jvm/JavaFlightRecorderTest.java | 72 +++++++++ 5 files changed, 360 insertions(+), 2 deletions(-) create mode 100644 spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/JavaFlightRecorder.java create mode 100644 spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java create mode 100644 spectator-ext-jvm/src/test/java/com/netflix/spectator/jvm/JavaFlightRecorderUnsupportedTest.java create mode 100644 spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java diff --git a/spectator-ext-jvm/build.gradle b/spectator-ext-jvm/build.gradle index e882d8741..d594be840 100644 --- a/spectator-ext-jvm/build.gradle +++ b/spectator-ext-jvm/build.gradle @@ -1,12 +1,58 @@ +sourceSets { + java17 { + java { + srcDirs = ['src/main/java17'] + compileClasspath = configurations.compileClasspath + runtimeClasspath = configurations.runtimeClasspath + } + } + java17Test { + java { + srcDirs = ['src/test/java17'] + compileClasspath = jar.outputs.files + configurations.testCompileClasspath + runtimeClasspath = jar.outputs.files + runtimeClasspath + configurations.testRuntimeClasspath + } + } +} + dependencies { api project(':spectator-api') implementation 'com.typesafe:config' } -jar { +def java17Compiler = javaToolchains.compilerFor { + languageVersion = JavaLanguageVersion.of(17) +} + +tasks.named('compileJava17Java', JavaCompile).configure { + javaCompiler = java17Compiler +} + +tasks.named('compileJava17TestJava', JavaCompile).configure { + javaCompiler = java17Compiler +} + +tasks.named('jar').configure { + into('META-INF/versions/17') { + from sourceSets.java17.output + } manifest { attributes( - "Automatic-Module-Name": "com.netflix.spectator.jvm" + 'Automatic-Module-Name': 'com.netflix.spectator.jvm', + 'Multi-Release': 'true' ) } } + +def testJava17 = tasks.register('testJava17', Test) { + description = "Runs tests for java17Test sourceset." + group = 'verification' + + testClassesDirs = sourceSets.java17Test.output.classesDirs + classpath = sourceSets.java17Test.runtimeClasspath + + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(17) + } +} +check.dependsOn testJava17 diff --git a/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/JavaFlightRecorder.java b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/JavaFlightRecorder.java new file mode 100644 index 000000000..695d574d7 --- /dev/null +++ b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/JavaFlightRecorder.java @@ -0,0 +1,53 @@ +/* + * Copyright 2014-2024 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spectator.jvm; + +import com.netflix.spectator.api.Registry; + +import java.util.concurrent.Executor; + +/** + * Helpers supporting continuous monitoring with Java Flight Recorder. + */ +public final class JavaFlightRecorder { + + private JavaFlightRecorder() { + } + + /** + * Return if Java Flight Recorder continuous monitoring is supported on the current JVM. + */ + public static boolean isSupported() { + return false; + } + + /** + * Collect low-overhead Java Flight Recorder events, using the provided + * {@link java.util.concurrent.Executor} to execute a single task to collect events. + *

+ * These measures provide parity with {@link Jmx#registerStandardMXBeans} and the + * `spectator-ext-gc` module. + * + * @param registry the registry + * @param executor the executor to execute the task for streaming events + * @return an {@link AutoCloseable} allowing the underlying event stream to be closed + */ + public static AutoCloseable monitorDefaultEvents(Registry registry, Executor executor) { + throw new UnsupportedOperationException("Java Flight Recorder support is only available on Java 17 and later"); + } + +} diff --git a/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java new file mode 100644 index 000000000..57f0d337a --- /dev/null +++ b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java @@ -0,0 +1,151 @@ +/* + * Copyright 2014-2024 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spectator.jvm; + +import com.netflix.spectator.api.Registry; +import jdk.jfr.EventSettings; +import jdk.jfr.consumer.RecordedEvent; +import jdk.jfr.consumer.RecordingStream; + +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +public class JavaFlightRecorder { + + private static final String PREFIX = "jdk."; + private static final String ClassLoadingStatistics = PREFIX + "ClassLoadingStatistics"; + private static final String CompilerStatistics = PREFIX + "CompilerStatistics"; + private static final String JavaThreadStatistics = PREFIX + "JavaThreadStatistics"; + private static final String VirtualThreadPinned = PREFIX + "VirtualThreadPinned"; + private static final String VirtualThreadSubmitFailed = PREFIX + "VirtualThreadSubmitFailed"; + private static final String ZAllocationStall = PREFIX + "ZAllocationStall"; + private static final String ZYoungGarbageCollection = PREFIX + "ZYoungGarbageCollection"; + private static final String ZOldGarbageCollection = PREFIX + "ZOldGarbageCollection"; + + private JavaFlightRecorder() { + } + + public static boolean isSupported() { + try { + Class.forName("jdk.jfr.consumer.RecordingStream"); + } catch (ClassNotFoundException e) { + return false; + } + return true; + } + + public static AutoCloseable monitorDefaultEvents(Registry registry, Executor executor) { + if (!isSupported()) { + throw new UnsupportedOperationException("This JVM does not support Java Flight Recorder event streaming"); + } + Objects.requireNonNull(registry); + Objects.requireNonNull(executor); + RecordingStream rs = new RecordingStream(); + collectClassLoadingStatistics(registry, rs); + collectCompilerStatistics(registry, rs); + collectThreadStatistics(registry, rs); + collectVirtualThreadEvents(registry, rs); + collectZgcEvents(registry, rs); + executor.execute(rs::start); + return rs::close; + } + + private static void collectClassLoadingStatistics(Registry registry, RecordingStream rs) { + AtomicLong prevLoadedClassCount = new AtomicLong(); + AtomicLong prevUnloadedClassCount = new AtomicLong(); + consume(ClassLoadingStatistics, rs, event -> { + long classesLoaded = event.getLong("loadedClassCount"); + classesLoaded = classesLoaded - prevLoadedClassCount.getAndSet(classesLoaded); + registry.counter("jvm.classloading.classesLoaded").increment(classesLoaded); + + long classesUnloaded = event.getLong("unloadedClassCount"); + classesUnloaded = classesUnloaded - prevUnloadedClassCount.getAndSet(classesUnloaded); + registry.counter("jvm.classloading.classesUnloaded").increment(classesUnloaded); + }); + } + + private static void collectCompilerStatistics(Registry registry, RecordingStream rs) { + AtomicLong prevTotalTimeSpent = new AtomicLong(); + consume(CompilerStatistics, rs, event -> { + long totalTimeSpent = event.getLong("totalTimeSpent"); + totalTimeSpent = totalTimeSpent - prevTotalTimeSpent.getAndAdd(totalTimeSpent); + registry.counter("jvm.compilation.compilationTime").add(totalTimeSpent / 1000.0); + }); + } + + private static void collectThreadStatistics(Registry registry, RecordingStream rs) { + AtomicLong prevAccumulatedCount = new AtomicLong(); + consume(JavaThreadStatistics, rs, event -> { + long activeCount = event.getLong("activeCount"); + long daemonCount = event.getLong("daemonCount"); + long nonDaemonCount = activeCount - daemonCount; + registry.gauge("jvm.thread.threadCount", "id", "non-daemon").set(nonDaemonCount); + registry.gauge("jvm.thread.threadCount", "id", "daemon").set(daemonCount); + long accumulatedCount = event.getLong("accumulatedCount"); + long threadsStarted = accumulatedCount - prevAccumulatedCount.getAndSet(accumulatedCount); + registry.counter("jvm.thread.threadsStarted").increment(threadsStarted); + }); + } + + private static void collectVirtualThreadEvents(Registry registry, RecordingStream rs) { + consume(VirtualThreadPinned, rs, event -> + registry.timer("jvm.vt.pinned").record(event.getDuration()) + ).withThreshold(Duration.ofMillis(20)); + consume(VirtualThreadSubmitFailed, rs, event -> + registry.counter("jvm.vt.submitFailed").increment() + ); + } + + private static void collectZgcEvents(Registry registry, RecordingStream rs) { + consume(ZYoungGarbageCollection, rs, event -> + registry.timer("jvm.zgc.youngCollection", "tenuringThreshold", event.getString("tenuringThreshold")) + .record(event.getDuration())); + + consume(ZOldGarbageCollection, rs, event -> + registry.timer("jvm.zgc.oldCollection") + .record(event.getDuration())); + + consume(ZAllocationStall, rs, event -> + registry.timer("jvm.zgc.allocationStall", "type", event.getString("type")) + .record(event.getDuration())); + } + + /** + * Consume a given JFR event. For full event details see the event definitions and default/profiling configuration: + *

+ * - metadata.xml + * - default.jfc + * - profile.jfc + *

+ * We avoid the default event configurations because despite their claims of "low-overhead" there are + * situtations where they can impose significant overhead to the application. + */ + private static EventSettings consume(String name, RecordingStream rs, Consumer consumer) { + // Apply sensible defaults to settings to avoid the overhead of collecting unnecessary stacktraces + // and collecting periodic events at a finer interval than we require upstream + EventSettings settings = rs.enable(name) + .withoutStackTrace() + .withThreshold(Duration.ofMillis(0)) + .withPeriod(Duration.ofSeconds(5)); + rs.onEvent(name, consumer); + return settings; + } + +} diff --git a/spectator-ext-jvm/src/test/java/com/netflix/spectator/jvm/JavaFlightRecorderUnsupportedTest.java b/spectator-ext-jvm/src/test/java/com/netflix/spectator/jvm/JavaFlightRecorderUnsupportedTest.java new file mode 100644 index 000000000..e54224db9 --- /dev/null +++ b/spectator-ext-jvm/src/test/java/com/netflix/spectator/jvm/JavaFlightRecorderUnsupportedTest.java @@ -0,0 +1,36 @@ +/* + * Copyright 2014-2024 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spectator.jvm; + +import com.netflix.spectator.api.NoopRegistry; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class JavaFlightRecorderUnsupportedTest { + + @Test + public void isUnsupported() { + Assertions.assertFalse(JavaFlightRecorder.isSupported()); + } + + @Test + public void monitorThrowsUOE() { + Assertions.assertThrows(UnsupportedOperationException.class, () -> + JavaFlightRecorder.monitorDefaultEvents(new NoopRegistry(), Runnable::run)); + } + +} diff --git a/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java b/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java new file mode 100644 index 000000000..b03d83489 --- /dev/null +++ b/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java @@ -0,0 +1,72 @@ +/* + * Copyright 2014-2024 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spectator.jvm; + +import com.netflix.spectator.api.DefaultRegistry; +import com.netflix.spectator.api.Id; +import com.netflix.spectator.api.Measurement; +import com.netflix.spectator.api.Registry; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; + +public class JavaFlightRecorderTest { + + @Test + public void isSupported() { + assertTrue(JavaFlightRecorder.isSupported()); + } + + @Test + public void checkDefaultMeasures() throws Exception { + Registry registry = new DefaultRegistry(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (var closable = JavaFlightRecorder.monitorDefaultEvents(registry, executor)) { + Thread.sleep(6000); + } + executor.shutdownNow(); + + Map measures = registry.measurements() + .collect(Collectors.toMap(Measurement::id, m -> m)); + + Measurement classesLoaded = measures.get(Id.create("jvm.classloading.classesLoaded")); + Measurement classesUnloaded = measures.get(Id.create("jvm.classloading.classesUnloaded")); + assertNotEquals(null, classesLoaded); + assertNotEquals(null, classesUnloaded); + assertTrue(classesLoaded.value() > 3000 && classesLoaded.value() < 4000); + assertEquals(0, classesUnloaded.value()); + + Measurement compilationTime = measures.get(Id.create("jvm.compilation.compilationTime")); + assertNotEquals(null, compilationTime); + + Measurement nonDaemonThreadCount = measures.get(Id.create("jvm.thread.threadCount").withTag("id", "non-daemon")); + Measurement daemonThreadCount = measures.get(Id.create("jvm.thread.threadCount").withTag("id", "daemon")); + Measurement threadsStarted = measures.get(Id.create("jvm.thread.threadsStarted")); + assertNotEquals(null, nonDaemonThreadCount); + assertEquals(5, nonDaemonThreadCount.value()); + assertNotEquals(null, daemonThreadCount); + assertEquals(7, daemonThreadCount.value()); + assertNotEquals(null, threadsStarted); + assertEquals(12, threadsStarted.value()); + } + +} From 81be494e9b0dffe762d70c1f3de67dadb468d698 Mon Sep 17 00:00:00 2001 From: Danny Thomas Date: Tue, 3 Sep 2024 00:21:22 +0000 Subject: [PATCH 2/8] Improve GC measures --- .../spectator/jvm/JavaFlightRecorder.java | 20 +++++++++---------- .../spectator/jvm/JavaFlightRecorderTest.java | 12 +++++++++++ 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java index 57f0d337a..cc92989c3 100644 --- a/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java +++ b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java @@ -35,9 +35,9 @@ public class JavaFlightRecorder { private static final String JavaThreadStatistics = PREFIX + "JavaThreadStatistics"; private static final String VirtualThreadPinned = PREFIX + "VirtualThreadPinned"; private static final String VirtualThreadSubmitFailed = PREFIX + "VirtualThreadSubmitFailed"; + private static final String YoungGarbageCollection = PREFIX + "YoungGarbageCollection"; private static final String ZAllocationStall = PREFIX + "ZAllocationStall"; private static final String ZYoungGarbageCollection = PREFIX + "ZYoungGarbageCollection"; - private static final String ZOldGarbageCollection = PREFIX + "ZOldGarbageCollection"; private JavaFlightRecorder() { } @@ -62,7 +62,7 @@ public static AutoCloseable monitorDefaultEvents(Registry registry, Executor exe collectCompilerStatistics(registry, rs); collectThreadStatistics(registry, rs); collectVirtualThreadEvents(registry, rs); - collectZgcEvents(registry, rs); + collectGcEvents(registry, rs); executor.execute(rs::start); return rs::close; } @@ -113,17 +113,15 @@ private static void collectVirtualThreadEvents(Registry registry, RecordingStrea ); } - private static void collectZgcEvents(Registry registry, RecordingStream rs) { - consume(ZYoungGarbageCollection, rs, event -> - registry.timer("jvm.zgc.youngCollection", "tenuringThreshold", event.getString("tenuringThreshold")) - .record(event.getDuration())); - - consume(ZOldGarbageCollection, rs, event -> - registry.timer("jvm.zgc.oldCollection") - .record(event.getDuration())); + private static void collectGcEvents(Registry registry, RecordingStream rs) { + Consumer tenuringThreshold = event -> + registry.gauge("jvm.gc.tenuringThreshold") + .set(event.getLong("tenuringThreshold")); + consume(YoungGarbageCollection, rs, tenuringThreshold); + consume(ZYoungGarbageCollection, rs, tenuringThreshold); consume(ZAllocationStall, rs, event -> - registry.timer("jvm.zgc.allocationStall", "type", event.getString("type")) + registry.timer("jvm.gc.allocationStall", "type", event.getString("type")) .record(event.getDuration())); } diff --git a/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java b/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java index b03d83489..40fbf3974 100644 --- a/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java +++ b/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java @@ -31,6 +31,8 @@ public class JavaFlightRecorderTest { + public static volatile Object obj; + @Test public void isSupported() { assertTrue(JavaFlightRecorder.isSupported()); @@ -41,6 +43,12 @@ public void checkDefaultMeasures() throws Exception { Registry registry = new DefaultRegistry(); ExecutorService executor = Executors.newSingleThreadExecutor(); try (var closable = JavaFlightRecorder.monitorDefaultEvents(registry, executor)) { + // allocate rapidly to trigger a GC, black holing using the approach from + // https://github.com/openjdk/jdk/blob/master/test/hotspot/jtreg/gc/testlibrary/Allocation.java + for (int i = 0; i < 100; i++) { + obj = new byte[4 * 1024 * 1024]; + obj = null; + } Thread.sleep(6000); } executor.shutdownNow(); @@ -48,6 +56,10 @@ public void checkDefaultMeasures() throws Exception { Map measures = registry.measurements() .collect(Collectors.toMap(Measurement::id, m -> m)); + Measurement tenuringThreshold = measures.get(Id.create("jvm.gc.tenuringThreshold")); + assertNotEquals(null, tenuringThreshold); + assertTrue(tenuringThreshold.value() > 0); + Measurement classesLoaded = measures.get(Id.create("jvm.classloading.classesLoaded")); Measurement classesUnloaded = measures.get(Id.create("jvm.classloading.classesUnloaded")); assertNotEquals(null, classesLoaded); From fdf7d82d00d9b2e5da29a749a68b776cda39e6b7 Mon Sep 17 00:00:00 2001 From: Danny Thomas Date: Tue, 3 Sep 2024 00:36:31 +0000 Subject: [PATCH 3/8] Avoid Spotbugs failure for intentional static write --- spectator-ext-jvm/build.gradle | 1 + .../com/netflix/spectator/jvm/JavaFlightRecorderTest.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/spectator-ext-jvm/build.gradle b/spectator-ext-jvm/build.gradle index d594be840..fdebd29ed 100644 --- a/spectator-ext-jvm/build.gradle +++ b/spectator-ext-jvm/build.gradle @@ -18,6 +18,7 @@ sourceSets { dependencies { api project(':spectator-api') implementation 'com.typesafe:config' + testImplementation 'com.google.code.findbugs:annotations:3.0.1u2' } def java17Compiler = javaToolchains.compilerFor { diff --git a/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java b/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java index 40fbf3974..48c344e70 100644 --- a/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java +++ b/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java @@ -20,6 +20,7 @@ import com.netflix.spectator.api.Id; import com.netflix.spectator.api.Measurement; import com.netflix.spectator.api.Registry; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.junit.jupiter.api.Test; import java.util.Map; @@ -31,6 +32,7 @@ public class JavaFlightRecorderTest { + @SuppressFBWarnings public static volatile Object obj; @Test From 5b74da6bfec5d82250b5f740d06790bb3eb25ad9 Mon Sep 17 00:00:00 2001 From: Danny Thomas Date: Tue, 3 Sep 2024 00:40:08 +0000 Subject: [PATCH 4/8] Use Flight Recorder when registerStandardMXBeans is called --- .../src/main/java/com/netflix/spectator/jvm/Jmx.java | 11 +++++++++++ .../netflix/spectator/jvm/JavaFlightRecorderTest.java | 4 ---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java index aec28f720..e28fa6dcf 100644 --- a/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java +++ b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java @@ -27,6 +27,8 @@ import java.lang.management.ManagementFactory; import java.lang.management.MemoryPoolMXBean; import java.lang.management.ThreadMXBean; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; /** * Helpers for working with JMX mbeans. @@ -42,6 +44,15 @@ private Jmx() { * mbeans from the local jvm. */ public static void registerStandardMXBeans(Registry registry) { + if (JavaFlightRecorder.isSupported()) { + Executor executor = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "spectator-jfr"); + t.setDaemon(true); + return t; + }); + JavaFlightRecorder.monitorDefaultEvents(registry, executor); + return; + } monitorClassLoadingMXBean(registry); monitorThreadMXBean(registry); monitorCompilationMXBean(registry); diff --git a/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java b/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java index 48c344e70..7b4642202 100644 --- a/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java +++ b/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java @@ -58,10 +58,6 @@ public void checkDefaultMeasures() throws Exception { Map measures = registry.measurements() .collect(Collectors.toMap(Measurement::id, m -> m)); - Measurement tenuringThreshold = measures.get(Id.create("jvm.gc.tenuringThreshold")); - assertNotEquals(null, tenuringThreshold); - assertTrue(tenuringThreshold.value() > 0); - Measurement classesLoaded = measures.get(Id.create("jvm.classloading.classesLoaded")); Measurement classesUnloaded = measures.get(Id.create("jvm.classloading.classesUnloaded")); assertNotEquals(null, classesLoaded); From 623e89c1a695772b66aef41779910412e4945715 Mon Sep 17 00:00:00 2001 From: brharrington Date: Tue, 3 Sep 2024 11:35:37 -0500 Subject: [PATCH 5/8] Update JavaFlightRecorder.java add comments --- .../java17/com/netflix/spectator/jvm/JavaFlightRecorder.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java index cc92989c3..54de9e8a2 100644 --- a/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java +++ b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.netflix.spectator.jvm; import com.netflix.spectator.api.Registry; @@ -105,6 +104,7 @@ private static void collectThreadStatistics(Registry registry, RecordingStream r } private static void collectVirtualThreadEvents(Registry registry, RecordingStream rs) { + // 20ms threshold set to match default behavior consume(VirtualThreadPinned, rs, event -> registry.timer("jvm.vt.pinned").record(event.getDuration()) ).withThreshold(Duration.ofMillis(20)); @@ -114,6 +114,9 @@ private static void collectVirtualThreadEvents(Registry registry, RecordingStrea } private static void collectGcEvents(Registry registry, RecordingStream rs) { + // ZGC and Shenandoah are not covered by the generic event, there is + // a ZGC specific event to get coverage there, right now there doesn't + // appear to be similar data available for Shenandoah Consumer tenuringThreshold = event -> registry.gauge("jvm.gc.tenuringThreshold") .set(event.getLong("tenuringThreshold")); From 23cd1b80efc05057bbd13d27ea729473fda76bcb Mon Sep 17 00:00:00 2001 From: Danny Thomas Date: Wed, 4 Sep 2024 00:50:38 +0000 Subject: [PATCH 6/8] Avoid accidentally dropping JMX-only metrics. Hoist meters out of consume lambdas --- .../java/com/netflix/spectator/jvm/Jmx.java | 7 ++-- .../spectator/jvm/JavaFlightRecorder.java | 41 +++++++++++-------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java index e28fa6dcf..35e16c220 100644 --- a/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java +++ b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java @@ -52,10 +52,11 @@ public static void registerStandardMXBeans(Registry registry) { }); JavaFlightRecorder.monitorDefaultEvents(registry, executor); return; + } else { + monitorClassLoadingMXBean(registry); + monitorThreadMXBean(registry); + monitorCompilationMXBean(registry); } - monitorClassLoadingMXBean(registry); - monitorThreadMXBean(registry); - monitorCompilationMXBean(registry); maybeRegisterHotspotInternal(registry); for (MemoryPoolMXBean mbean : ManagementFactory.getMemoryPoolMXBeans()) { diff --git a/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java index 54de9e8a2..6ca99d02d 100644 --- a/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java +++ b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java @@ -15,7 +15,10 @@ */ package com.netflix.spectator.jvm; +import com.netflix.spectator.api.Counter; +import com.netflix.spectator.api.Gauge; import com.netflix.spectator.api.Registry; +import com.netflix.spectator.api.Timer; import jdk.jfr.EventSettings; import jdk.jfr.consumer.RecordedEvent; import jdk.jfr.consumer.RecordingStream; @@ -67,29 +70,33 @@ public static AutoCloseable monitorDefaultEvents(Registry registry, Executor exe } private static void collectClassLoadingStatistics(Registry registry, RecordingStream rs) { + Counter classesLoaded = registry.counter("jvm.classloading.classesLoaded"); AtomicLong prevLoadedClassCount = new AtomicLong(); + Counter classesUnloaded = registry.counter("jvm.classloading.classesUnloaded"); AtomicLong prevUnloadedClassCount = new AtomicLong(); consume(ClassLoadingStatistics, rs, event -> { - long classesLoaded = event.getLong("loadedClassCount"); - classesLoaded = classesLoaded - prevLoadedClassCount.getAndSet(classesLoaded); - registry.counter("jvm.classloading.classesLoaded").increment(classesLoaded); + long loadedClassCount = event.getLong("loadedClassCount"); + loadedClassCount = loadedClassCount - prevLoadedClassCount.getAndSet(loadedClassCount); + classesLoaded.increment(loadedClassCount); - long classesUnloaded = event.getLong("unloadedClassCount"); - classesUnloaded = classesUnloaded - prevUnloadedClassCount.getAndSet(classesUnloaded); - registry.counter("jvm.classloading.classesUnloaded").increment(classesUnloaded); + long unloadedClassCount = event.getLong("unloadedClassCount"); + unloadedClassCount = unloadedClassCount - prevUnloadedClassCount.getAndSet(unloadedClassCount); + classesUnloaded.increment(unloadedClassCount); }); } private static void collectCompilerStatistics(Registry registry, RecordingStream rs) { + Counter compilationTime = registry.counter("jvm.compilation.compilationTime"); AtomicLong prevTotalTimeSpent = new AtomicLong(); consume(CompilerStatistics, rs, event -> { long totalTimeSpent = event.getLong("totalTimeSpent"); totalTimeSpent = totalTimeSpent - prevTotalTimeSpent.getAndAdd(totalTimeSpent); - registry.counter("jvm.compilation.compilationTime").add(totalTimeSpent / 1000.0); + compilationTime.add(totalTimeSpent / 1000.0); }); } private static void collectThreadStatistics(Registry registry, RecordingStream rs) { + Counter threadsStarted = registry.counter("jvm.thread.threadsStarted"); AtomicLong prevAccumulatedCount = new AtomicLong(); consume(JavaThreadStatistics, rs, event -> { long activeCount = event.getLong("activeCount"); @@ -98,18 +105,20 @@ private static void collectThreadStatistics(Registry registry, RecordingStream r registry.gauge("jvm.thread.threadCount", "id", "non-daemon").set(nonDaemonCount); registry.gauge("jvm.thread.threadCount", "id", "daemon").set(daemonCount); long accumulatedCount = event.getLong("accumulatedCount"); - long threadsStarted = accumulatedCount - prevAccumulatedCount.getAndSet(accumulatedCount); - registry.counter("jvm.thread.threadsStarted").increment(threadsStarted); + accumulatedCount = accumulatedCount - prevAccumulatedCount.getAndSet(accumulatedCount); + threadsStarted.increment(accumulatedCount); }); } private static void collectVirtualThreadEvents(Registry registry, RecordingStream rs) { + Timer pinned = registry.timer("jvm.vt.pinned"); + Counter submitFailed = registry.counter("jvm.vt.submitFailed"); // 20ms threshold set to match default behavior consume(VirtualThreadPinned, rs, event -> - registry.timer("jvm.vt.pinned").record(event.getDuration()) + pinned.record(event.getDuration()) ).withThreshold(Duration.ofMillis(20)); consume(VirtualThreadSubmitFailed, rs, event -> - registry.counter("jvm.vt.submitFailed").increment() + submitFailed.increment() ); } @@ -117,11 +126,11 @@ private static void collectGcEvents(Registry registry, RecordingStream rs) { // ZGC and Shenandoah are not covered by the generic event, there is // a ZGC specific event to get coverage there, right now there doesn't // appear to be similar data available for Shenandoah - Consumer tenuringThreshold = event -> - registry.gauge("jvm.gc.tenuringThreshold") - .set(event.getLong("tenuringThreshold")); - consume(YoungGarbageCollection, rs, tenuringThreshold); - consume(ZYoungGarbageCollection, rs, tenuringThreshold); + Gauge tenuringThreshold = registry.gauge("jvm.gc.tenuringThreshold"); + Consumer tenuringThresholdFn = event -> + tenuringThreshold.set(event.getLong("tenuringThreshold")); + consume(YoungGarbageCollection, rs, tenuringThresholdFn); + consume(ZYoungGarbageCollection, rs, tenuringThresholdFn); consume(ZAllocationStall, rs, event -> registry.timer("jvm.gc.allocationStall", "type", event.getString("type")) From 37d39fc399df1772e48f725f2d9e49919e7ba301 Mon Sep 17 00:00:00 2001 From: brharrington Date: Tue, 3 Sep 2024 21:03:06 -0500 Subject: [PATCH 7/8] Update Jmx.java make indentation consistent --- .../src/main/java/com/netflix/spectator/jvm/Jmx.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java index 35e16c220..6e11fa99b 100644 --- a/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java +++ b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java @@ -53,9 +53,9 @@ public static void registerStandardMXBeans(Registry registry) { JavaFlightRecorder.monitorDefaultEvents(registry, executor); return; } else { - monitorClassLoadingMXBean(registry); - monitorThreadMXBean(registry); - monitorCompilationMXBean(registry); + monitorClassLoadingMXBean(registry); + monitorThreadMXBean(registry); + monitorCompilationMXBean(registry); } maybeRegisterHotspotInternal(registry); From c28bb9fc3c67bc3e7b39352933c44b2b90738a7e Mon Sep 17 00:00:00 2001 From: brharrington Date: Tue, 3 Sep 2024 21:06:53 -0500 Subject: [PATCH 8/8] Update JavaFlightRecorder.java Hold reference to thread count gauges --- .../com/netflix/spectator/jvm/JavaFlightRecorder.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java index 6ca99d02d..d92cba9ca 100644 --- a/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java +++ b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java @@ -96,14 +96,16 @@ private static void collectCompilerStatistics(Registry registry, RecordingStream } private static void collectThreadStatistics(Registry registry, RecordingStream rs) { + Gauge nonDaemonThreadCount = registry.gauge("jvm.thread.threadCount", "id", "non-daemon"); + Gauge daemonThreadCount = registry.gauge("jvm.thread.threadCount", "id", "daemon"); Counter threadsStarted = registry.counter("jvm.thread.threadsStarted"); AtomicLong prevAccumulatedCount = new AtomicLong(); consume(JavaThreadStatistics, rs, event -> { long activeCount = event.getLong("activeCount"); long daemonCount = event.getLong("daemonCount"); long nonDaemonCount = activeCount - daemonCount; - registry.gauge("jvm.thread.threadCount", "id", "non-daemon").set(nonDaemonCount); - registry.gauge("jvm.thread.threadCount", "id", "daemon").set(daemonCount); + nonDaemonThreadCount.set(nonDaemonCount); + daemonThreadCount.set(daemonCount); long accumulatedCount = event.getLong("accumulatedCount"); accumulatedCount = accumulatedCount - prevAccumulatedCount.getAndSet(accumulatedCount); threadsStarted.increment(accumulatedCount); @@ -126,8 +128,8 @@ private static void collectGcEvents(Registry registry, RecordingStream rs) { // ZGC and Shenandoah are not covered by the generic event, there is // a ZGC specific event to get coverage there, right now there doesn't // appear to be similar data available for Shenandoah - Gauge tenuringThreshold = registry.gauge("jvm.gc.tenuringThreshold"); - Consumer tenuringThresholdFn = event -> + Gauge tenuringThreshold = registry.gauge("jvm.gc.tenuringThreshold"); + Consumer tenuringThresholdFn = event -> tenuringThreshold.set(event.getLong("tenuringThreshold")); consume(YoungGarbageCollection, rs, tenuringThresholdFn); consume(ZYoungGarbageCollection, rs, tenuringThresholdFn);