Merge branch 'desyncr-thread-diagnostics-cpu' into next

src/freenet/clients/http/DiagnosticToadlet.java

@@ -2,12 +2,16 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.management.*;
 import java.net.URI;
 import java.text.DecimalFormat;
 import java.text.NumberFormat;
 import java.util.Arrays;
 import java.util.Comparator;
+import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.*;
 
 import freenet.client.HighLevelSimpleClient;
 import freenet.client.async.PersistenceDisabledException;
@@ -29,6 +33,8 @@
 import freenet.node.PeerNodeStatus;
 import freenet.node.RequestTracker;
 import freenet.node.Version;
+import freenet.node.diagnostics.*;
+import freenet.node.diagnostics.threads.*;
 import freenet.node.stats.DataStoreInstanceType;
 import freenet.node.stats.DataStoreStats;
 import freenet.node.stats.StatsNotAvailableException;
@@ -410,19 +416,69 @@ public int compare(PeerNodeStatus firstNode, PeerNodeStatus secondNode) {
 		textBuilder.append("\n");
 
 		// drawThreadPriorityStatsBox
-		textBuilder.append("Threads:\n");
-		int[] activeThreadsByPriority = stats.getActiveThreadsByPriority();
-		int[] waitingThreadsByPriority = stats.getWaitingThreadsByPriority();
-		for(int i=0; i<activeThreadsByPriority.length; i++) {
-			textBuilder.append(l10n("running")).append(": ").append(String.valueOf(activeThreadsByPriority[i])).append(" (").append(String.valueOf(i+1)).append(")\n");
-			textBuilder.append(l10n("waiting")).append(": ").append(String.valueOf(waitingThreadsByPriority[i])).append(" (").append(String.valueOf(i+1)).append(")\n");
+		if (node.isNodeDiagnosticsEnabled()) {
+			textBuilder.append(threadsStats());
+			textBuilder.append("\n");
 		}
-		textBuilder.append("\n");
 		}
 
 		this.writeTextReply(ctx, 200, "OK", textBuilder.toString());
 	}
 
+	/**
+	 * Retrieves ThreadDiagnostics (through NodeDiagnostics) to display
+	 * thread information (id, name, group, % cpu, etc).
+	 * @return Thread information in tab separated format.
+	 */
+	private StringBuilder threadsStats() {
+		StringBuilder sb = new StringBuilder();
+
+		ThreadDiagnostics threadDiagnostics = node
+			.getNodeDiagnostics()
+			.getThreadDiagnostics();
+
+		NodeThreadSnapshot threadSnapshot = threadDiagnostics.getThreadSnapshot();
+
+		double wallTime = TimeUnit.MILLISECONDS.toNanos(
+			threadSnapshot.getInterval()
+		);
+
+		List<NodeThreadInfo> threads = threadSnapshot.getThreads();
+		threads.sort(Comparator.comparing(NodeThreadInfo::getCpuTime).reversed());
+
+		sb.append(String.format("Threads (%d):%n", threads.size()));
+
+		// Thread ID, Job ID, Name, Priority, Group (system, main), Status, % CPU
+		sb.append(
+			String.format(
+				"%10s %15s %-90s %5s %10s %-20s %-5s%n",
+				"Thread ID",
+				"Job ID",
+				"Name",
+				"Prio.",
+				"Group",
+				"Status",
+				"% CPU"
+			)
+		);
+
+		for (NodeThreadInfo thread : threads) {
+			String line = String.format(
+				"%10s %15s %-90s %5s %10s %-20s %.2f%n",
+				thread.getId(),
+				thread.getJobId(),
+				thread.getName().substring(0, Math.min(90, thread.getName().length())),
+				thread.getPrio(),
+				thread.getGroupName().substring(0, Math.min(10, thread.getGroupName().length())),
+				thread.getState(),
+				thread.getCpuTime() / wallTime * 100
+			);
+			sb.append(line);
+		}
+
+		return sb;
+	}
+
 	private int getPeerStatusCount(PeerNodeStatus[] peerNodeStatuses, int status) {
 		int count = 0;
 		for (PeerNodeStatus peerNodeStatus: peerNodeStatuses) {
@@ -471,4 +527,4 @@ private String l10n(String key, String[] patterns, String[] values) {
 	public String path() {
 		return TOADLET_URL;
 	}
-}
+}

src/freenet/l10n/freenet.l10n.en.properties

@@ -1105,6 +1105,8 @@ Node.enablePerNodeFailureTables=Enable per-node failure tables?
 Node.enablePerNodeFailureTablesLong=Enable automatically rerouting around nodes that failed a request within the last 10 minutes?
 Node.enableRoutedPing=Enable FNPRoutedPing?
 Node.enableRoutedPingLong=Enable FNPRoutedPing? Only useful in simulations, not on the real network. Turn it off.
+Node.enableDiagnostics=Enable Diagnostics?
+Node.enableDiagnosticsLong=By enabling Diagnostics the node will keep detailed information of its inner-workings (such as CPU usage per thread) which can help to troubleshoot problems. The collected data is kept in memory (i.e. not persisted on disk) and it's not sent to anybody over the network.
 Node.enableSwapping=Enable location swapping?
 Node.enableSwappingLong=Enable location swapping?
 Node.enableSwapQueueing=Enable queueing of swap requests?

src/freenet/node/Node.java

@@ -38,6 +38,7 @@
 import java.util.Set;
 
 import freenet.config.*;
+import freenet.node.diagnostics.*;
 import freenet.node.useralerts.*;
 import org.tanukisoftware.wrapper.WrapperManager;
 
@@ -724,6 +725,9 @@ public String[] getPossibleValues() {
 
 	public final SecurityLevels securityLevels;
 
+	/** Diagnostics */
+	private final DefaultNodeDiagnostics nodeDiagnostics;
+
 	// Things that's needed to keep track of
 	public final PluginManager pluginManager;
 
@@ -751,6 +755,8 @@ public String[] getPossibleValues() {
 
 	private boolean enableRoutedPing;
 
+	private boolean enableNodeDiagnostics;
+
 	private boolean peersOffersDismissed;
 
 	/**
@@ -2538,7 +2544,38 @@ public void set(Boolean val) throws InvalidConfigValueException,
 
 		});
 		enableRoutedPing = nodeConfig.getBoolean("enableRoutedPing");
-
+
+		nodeConfig.register(
+			"enableNodeDiagnostics",
+			false,
+			sortOrder++,
+			true,
+			false,
+			"Node.enableDiagnostics",
+			"Node.enableDiagnosticsLong",
+			new BooleanCallback() {
+				@Override
+				public Boolean get() {
+					synchronized (Node.this) {
+						return enableNodeDiagnostics;
+					}
+				}
+
+				@Override
+				public void set(Boolean val) {
+					synchronized (Node.this) {
+						enableNodeDiagnostics = val;
+						nodeDiagnostics.stop();
+
+						if (enableNodeDiagnostics) {
+							nodeDiagnostics.start();
+						}
+					}
+				}
+			}
+		);
+		enableNodeDiagnostics = nodeConfig.getBoolean("enableNodeDiagnostics");
+
 		updateMTU();
 
 		// peers-offers/*.fref files
@@ -2603,6 +2640,8 @@ public void realRun() {
 		System.out.println("Node constructor completed");
 
 		new BandwidthManager(this).start();
+
+		nodeDiagnostics = new DefaultNodeDiagnostics(this.nodeStats, this.ticker);
 	}
 
 	private void peersOffersFrefFilesConfiguration(SubConfig nodeConfig, int configOptionSortOrder) {
@@ -3168,6 +3207,10 @@ public void start(boolean noSwaps) throws NodeInitException {
 		// Process any data in the extra peer data directory
 		peers.readExtraPeerData();
 
+		if (enableNodeDiagnostics) {
+			nodeDiagnostics.start();
+		}
+
 		Logger.normal(this, "Started node");
 
 		hasStarted = true;
@@ -4894,5 +4937,12 @@ public PluginManager getPluginManager() {
     DatabaseKey getDatabaseKey() {
         return databaseKey;
     }
-
+
+    public NodeDiagnostics getNodeDiagnostics() {
+        return nodeDiagnostics;
+    }
+
+    public boolean isNodeDiagnosticsEnabled() {
+        return enableNodeDiagnostics;
+    }
 }

src/freenet/node/diagnostics/DefaultNodeDiagnostics.java

@@ -0,0 +1,46 @@
+/* This code is part of Freenet. It is distributed under the GNU General
+ * Public License, version 2 (or at your option any later version). See
+ * http://www.gnu.org/ for further details of the GPL. */
+package freenet.node.diagnostics;
+
+import freenet.node.diagnostics.threads.*;
+import freenet.support.Ticker;
+import freenet.node.NodeStats;
+
+/**
+ *  @author desyncr
+ *
+ *  A class to retrieve data to build diagnostic dumps to help in determining
+ *  node bottlenecks or misconfiguration.
+ *
+ *  This class launches various threads at intervals to retrieve information. This information
+ *  is available through the public methods.
+ *  Some data pointers are obtained from NodeStats object.
+ */
+public class DefaultNodeDiagnostics implements NodeDiagnostics {
+    private final DefaultThreadDiagnostics defaultThreadDiagnostics;
+
+   /**
+     * @param nodeStats Used to retrieve data points.
+     * @param ticker Used to queue timed jobs.
+     */
+    public DefaultNodeDiagnostics(NodeStats nodeStats, Ticker ticker) {
+        defaultThreadDiagnostics = new DefaultThreadDiagnostics(nodeStats, ticker);
+    }
+
+    public void start() {
+        defaultThreadDiagnostics.start();
+    }
+
+    public void stop() {
+        defaultThreadDiagnostics.stop();
+    }
+
+    /**
+     * @return List of threads registered in NodeStats.getThreads()
+     */
+    @Override
+    public ThreadDiagnostics getThreadDiagnostics() {
+        return defaultThreadDiagnostics;
+    }
+}

src/freenet/node/diagnostics/NodeDiagnostics.java

@@ -0,0 +1,10 @@
+/* This code is part of Freenet. It is distributed under the GNU General
+ * Public License, version 2 (or at your option any later version). See
+ * http://www.gnu.org/ for further details of the GPL. */
+package freenet.node.diagnostics;
+
+import freenet.node.diagnostics.threads.*;
+
+public interface NodeDiagnostics {
+    ThreadDiagnostics getThreadDiagnostics();
+}

src/freenet/node/diagnostics/ThreadDiagnostics.java

@@ -0,0 +1,10 @@
+/* This code is part of Freenet. It is distributed under the GNU General
+ * Public License, version 2 (or at your option any later version). See
+ * http://www.gnu.org/ for further details of the GPL. */
+package freenet.node.diagnostics;
+
+import freenet.node.diagnostics.threads.*;
+
+public interface ThreadDiagnostics {
+    NodeThreadSnapshot getThreadSnapshot();
+}