#1830 Health and Diagnostic Monitoring

src/main/java/io/github/dsheirer/gui/SDRTrunk.java

@@ -189,10 +189,13 @@ public SDRTrunk()
         EventLogManager eventLogManager = new EventLogManager(aliasModel, mUserPreferences);
         mPlaylistManager = new PlaylistManager(mUserPreferences, mTunerManager, aliasModel, eventLogManager, mIconModel);
 
+        boolean headless = GraphicsEnvironment.isHeadless();
+
         mDiagnosticMonitor = new DiagnosticMonitor(mUserPreferences, mPlaylistManager.getChannelProcessingManager(),
-                mTunerManager);
+                mTunerManager, headless);
+        mDiagnosticMonitor.start();
 
-        if(!GraphicsEnvironment.isHeadless())
+        if(!headless)
         {
             mJavaFxWindowManager = new JavaFxWindowManager(mUserPreferences, mTunerManager, mPlaylistManager);
         }
@@ -420,7 +423,7 @@ private void initGUI()
         processingStatusReportMenuItem.addActionListener(e -> {
             try
             {
-                Path path = mDiagnosticMonitor.generateProcessingDiagnosticReport();
+                Path path = mDiagnosticMonitor.generateProcessingDiagnosticReport("User initiated diagnostic report");
 
                 JOptionPane.showMessageDialog(mMainGui, "Report created: " +
                         path.toString(), "Processing Status Report Created", JOptionPane.INFORMATION_MESSAGE);
@@ -623,6 +626,7 @@ private void initGUI()
     private void processShutdown()
     {
         mLog.info("Application shutdown started ...");
+        mDiagnosticMonitor.stop();
         mUserPreferences.getSwingPreference().setLocation(WINDOW_FRAME_IDENTIFIER, mMainGui.getLocation());
         mUserPreferences.getSwingPreference().setDimension(WINDOW_FRAME_IDENTIFIER, mMainGui.getSize());
         mUserPreferences.getSwingPreference().setMaximized(WINDOW_FRAME_IDENTIFIER,

src/main/java/io/github/dsheirer/gui/preference/application/ApplicationPreferenceEditor.java

@@ -1,6 +1,6 @@
 /*
  * *****************************************************************************
- * Copyright (C) 2014-2023 Dennis Sheirer
+ * Copyright (C) 2014-2024 Dennis Sheirer
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -23,10 +23,16 @@
 import io.github.dsheirer.preference.application.ApplicationPreference;
 import javafx.geometry.HPos;
 import javafx.geometry.Insets;
+import javafx.geometry.Orientation;
 import javafx.scene.control.Label;
+import javafx.scene.control.Separator;
 import javafx.scene.control.Spinner;
+import javafx.scene.layout.ColumnConstraints;
 import javafx.scene.layout.GridPane;
 import javafx.scene.layout.HBox;
+import javafx.scene.layout.Priority;
+import javafx.scene.layout.VBox;
+import org.controlsfx.control.ToggleSwitch;
 
 
 /**
@@ -38,6 +44,7 @@ public class ApplicationPreferenceEditor extends HBox
     private GridPane mEditorPane;
     private Label mAutoStartTimeoutLabel;
     private Spinner<Integer> mTimeoutSpinner;
+    private ToggleSwitch mAutomaticDiagnosticMonitoringToggle;
 
     /**
      * Constructs an instance
@@ -46,7 +53,14 @@ public class ApplicationPreferenceEditor extends HBox
     public ApplicationPreferenceEditor(UserPreferences userPreferences)
     {
         mApplicationPreference = userPreferences.getApplicationPreference();
-        getChildren().add(getEditorPane());
+        setMaxWidth(Double.MAX_VALUE);
+
+        VBox vbox = new VBox();
+        vbox.setMaxHeight(Double.MAX_VALUE);
+        vbox.setMaxWidth(Double.MAX_VALUE);
+        vbox.getChildren().add(getEditorPane());
+        HBox.setHgrow(vbox, Priority.ALWAYS);
+        getChildren().add(vbox);
     }
 
     private GridPane getEditorPane()
@@ -55,13 +69,31 @@ private GridPane getEditorPane()
         {
             int row = 0;
             mEditorPane = new GridPane();
+            mEditorPane.setMaxWidth(Double.MAX_VALUE);
             mEditorPane.setVgap(10);
-            mEditorPane.setHgap(10);
+            mEditorPane.setHgap(3);
             mEditorPane.setPadding(new Insets(10, 10, 10, 10));
-            GridPane.setHalignment(getAutoStartTimeoutLabel(), HPos.RIGHT);
-            mEditorPane.add(getAutoStartTimeoutLabel(), 0, row);
-            mEditorPane.add(getTimeoutSpinner(), 1, row);
-            mEditorPane.add(new Label("seconds"), 2, row);
+
+            Label monitoringLabel = new Label("Application Health and Diagnostic Monitoring.");
+            mEditorPane.add(monitoringLabel, 0, row, 2, 1);
+            GridPane.setHalignment(getAutomaticDiagnosticMonitoringToggle(), HPos.RIGHT);
+            mEditorPane.add(getAutomaticDiagnosticMonitoringToggle(), 0, ++row);
+            mEditorPane.add(new Label("Enable Diagnostic Monitoring"), 1, row, 2, 1);
+
+            Separator separator = new Separator(Orientation.HORIZONTAL);
+            GridPane.setHgrow(separator, Priority.ALWAYS);
+            mEditorPane.add(separator, 0, ++row, 3, 1);
+
+            mEditorPane.add(getAutoStartTimeoutLabel(), 0, ++row, 2, 1);
+            GridPane.setHalignment(getTimeoutSpinner(), HPos.RIGHT);
+            mEditorPane.add(getTimeoutSpinner(), 0, ++row);
+            mEditorPane.add(new Label("seconds"), 1, row);
+
+            ColumnConstraints c1 = new ColumnConstraints();
+            c1.setPercentWidth(30);
+            ColumnConstraints c2 = new ColumnConstraints();
+            c2.setHgrow(Priority.ALWAYS);
+            mEditorPane.getColumnConstraints().addAll(c1, c2);
         }
 
         return mEditorPane;
@@ -91,4 +123,20 @@ private Spinner<Integer> getTimeoutSpinner()
 
         return mTimeoutSpinner;
     }
+
+    /**
+     * Toggle switch to enable/disable automatic diagnostic monitoring.
+     */
+    private ToggleSwitch getAutomaticDiagnosticMonitoringToggle()
+    {
+        if(mAutomaticDiagnosticMonitoringToggle == null)
+        {
+            mAutomaticDiagnosticMonitoringToggle = new ToggleSwitch();
+            mAutomaticDiagnosticMonitoringToggle.setSelected(mApplicationPreference.isAutomaticDiagnosticMonitoring());
+            mAutomaticDiagnosticMonitoringToggle.selectedProperty().addListener((observable, oldValue, enabled) ->
+                    mApplicationPreference.setAutomaticDiagnosticMonitoring(enabled));
+        }
+
+        return mAutomaticDiagnosticMonitoringToggle;
+    }
 }

src/main/java/io/github/dsheirer/monitor/DiagnosticMonitor.java

@@ -20,52 +20,165 @@
 package io.github.dsheirer.monitor;
 
 import io.github.dsheirer.controller.channel.ChannelProcessingManager;
+import io.github.dsheirer.log.LoggingSuppressor;
 import io.github.dsheirer.preference.UserPreferences;
 import io.github.dsheirer.source.tuner.manager.TunerManager;
+import io.github.dsheirer.util.ThreadPool;
 import io.github.dsheirer.util.TimeStamp;
 import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Enumeration;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
 import java.util.jar.Attributes;
 import java.util.jar.Manifest;
 import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.swing.JOptionPane;
+
 /**
  * Utility class for monitoring system components and producing logging reports.
  */
 public class DiagnosticMonitor
 {
     private static final Logger LOGGER = LoggerFactory.getLogger(DiagnosticMonitor.class);
+    private final LoggingSuppressor LOG_SUPPRESSOR = new LoggingSuppressor(LOGGER);
     private static final String DIVIDER = "\n\n=========================================================================\n\n";
     private UserPreferences mUserPreferences;
     private ChannelProcessingManager mChannelProcessingManager;
     private TunerManager mTunerManager;
+    private ScheduledFuture<?> mBlockedThreadMonitorHandle;
+    private BlockedThreadMonitor mMonitor = new BlockedThreadMonitor();
+    private boolean mUserAlertedToBlockedThreadCondition = false;
+    private Map<Integer,Integer> mBlockedThreadDetectionCountMap = new HashMap<>();
+    private boolean mHeadless;
 
     /**
      * Constructs an instance
      * @param userPreferences for application logging directory lookup.
+     * @param channelProcessingManager for accessing running channel information
+     * @param tunerManager for accessing allocated tuner channel information
+     * @param headless to indicate if the thread deadlock monitor should show a user notification.
      */
     public DiagnosticMonitor(UserPreferences userPreferences, ChannelProcessingManager channelProcessingManager,
-                             TunerManager tunerManager)
+                             TunerManager tunerManager, boolean headless)
     {
         mUserPreferences = userPreferences;
         mChannelProcessingManager = channelProcessingManager;
         mTunerManager = tunerManager;
+        mHeadless = headless;
+    }
+
+    /**
+     * Starts monitoring for blocked threads
+     */
+    public void start()
+    {
+        if(mBlockedThreadMonitorHandle != null)
+        {
+            mBlockedThreadMonitorHandle.cancel(true);
+        }
+
+        if(mUserPreferences.getApplicationPreference().isAutomaticDiagnosticMonitoring())
+        {
+            LOGGER.info("Diagnostic monitoring enabled running every 30 seconds");
+            mBlockedThreadMonitorHandle = ThreadPool.SCHEDULED.scheduleAtFixedRate(mMonitor, 30, 30, TimeUnit.SECONDS);
+        }
+        else
+        {
+            LOGGER.info("Diagnostic monitoring disabled per user preference (application).");
+        }
+    }
+
+    /**
+     * Stops monitoring for blocked threads.
+     */
+    public void stop()
+    {
+        if(mBlockedThreadMonitorHandle != null)
+        {
+            mBlockedThreadMonitorHandle.cancel(true);
+        }
+
+        mBlockedThreadMonitorHandle = null;
+    }
+
+    /**
+     * Checks for blocked threads and on discovery, generates a diagnostic report and notifies the user (once).
+     */
+    private void checkForBlockedThreads()
+    {
+        if(!mUserAlertedToBlockedThreadCondition)
+        {
+            try
+            {
+                ThreadMXBean bean = ManagementFactory.getThreadMXBean();
+
+                long ids[] = bean.findDeadlockedThreads();
+
+                if(ids != null)
+                {
+                    mUserAlertedToBlockedThreadCondition = true;
+
+                    ThreadInfo threadInfo[] = bean.getThreadInfo(ids);
+
+                    StringBuilder sb = new StringBuilder();
+                    sb.append("sdrtrunk detected a critical application error with a threading deadlock, described as follows:\n");
+
+                    for (ThreadInfo threadInfo1 : threadInfo)
+                    {
+                        sb.append("Thread ID[").append(threadInfo1.getThreadId());
+                        sb.append("] Name [").append(threadInfo1.getThreadName());
+                        sb.append("] Lock [").append(threadInfo1.getLockName());
+                        sb.append("] Owned By [ID:").append(threadInfo1.getLockOwnerId());
+                        sb.append(" | NAME:").append(threadInfo1.getLockName());
+                        sb.append("]\n");
+                    }
+
+                    LOGGER.error(sb.toString());
+                    Path reportPath = generateProcessingDiagnosticReport(sb + DIVIDER);
+                    LOGGER.error("Thread deadlock report generated: " + reportPath);
+
+                    if(!mHeadless)
+                    {
+                        String title = "sdrtrunk: Critical Error Detected";
+                        String message = "The sdrtrunk application has detected a thread deadlock situation.\n" +
+                                         "The application may degrade over time and eventually run out of memory.\n" +
+                                         "A diagnostic report was generated.  Please open an issue on the GitHub\n" +
+                                         "website and attach this diagnostic report:\n\n" + reportPath.toString();
+                        JOptionPane.showMessageDialog(null, message, title, JOptionPane.ERROR_MESSAGE);
+                    }
+                }
+            }
+            catch(Exception e)
+            {
+                LOG_SUPPRESSOR.error("run error", 1, "Error while monitoring for deadlocked " +
+                        "threads: " + e.getLocalizedMessage());
+                //Set the flag so that we don't try to run again.
+                mUserAlertedToBlockedThreadCondition = true;
+            }
+        }
     }
 
     /**
      * Creates a diagnostic report containing state information for channels that are in a processing state.
+     * @param message to prepend to the report
      * @return path for the log file that was created.
      */
-    public Path generateProcessingDiagnosticReport() throws IOException
+    public Path generateProcessingDiagnosticReport(String message) throws IOException
     {
         StringBuilder sb = new StringBuilder();
-        sb.append("sdrtrunk Processing Diagnostic Report\n");
+        sb.append(message);
+        sb.append("\n\nsdrtrunk Processing Diagnostic Report\n");
         sb.append(DIVIDER);
         sb.append(getEnvironmentReport());
         sb.append(DIVIDER);
@@ -190,4 +303,23 @@ public Attributes findManifestAttributes() {
 
         return null;
     }
+
+    /**
+     * Runnable to periodically check for blocked threads
+     */
+    public class BlockedThreadMonitor implements Runnable
+    {
+        @Override
+        public void run()
+        {
+            try
+            {
+                checkForBlockedThreads();
+            }
+            catch(Throwable t)
+            {
+                LOG_SUPPRESSOR.error("Error", 3, "Error while checking for blocked threads", t);
+            }
+        }
+    }
 }

src/main/java/io/github/dsheirer/preference/application/ApplicationPreference.java

@@ -1,6 +1,6 @@
 /*
  * *****************************************************************************
- * Copyright (C) 2014-2023 Dennis Sheirer
+ * Copyright (C) 2014-2024 Dennis Sheirer
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -31,11 +31,13 @@
  */
 public class ApplicationPreference extends Preference
 {
+    private static final String PREFERENCE_KEY_CHANNEL_AUTO_DIAGNOSTIC_MONITORING = "automatic.diagnostic.monitoring";
     private static final String PREFERENCE_KEY_CHANNEL_AUTO_START_TIMEOUT = "channel.auto.start.timeout";
 
     private final static Logger mLog = LoggerFactory.getLogger(ApplicationPreference.class);
     private Preferences mPreferences = Preferences.userNodeForPackage(ApplicationPreference.class);
     private Integer mChannelAutoStartTimeout;
+    private Boolean mAutomaticDiagnosticMonitoring;
 
     /**
      * Constructs an instance
@@ -77,4 +79,29 @@ public void setChannelAutoStartTimeout(int timeout)
         mPreferences.putInt(PREFERENCE_KEY_CHANNEL_AUTO_START_TIMEOUT, timeout);
         notifyPreferenceUpdated();
     }
+
+    /**
+     * Indicates if automatic diagnostic monitoring is enabled.
+     * @return enabled.
+     */
+    public boolean isAutomaticDiagnosticMonitoring()
+    {
+        if(mAutomaticDiagnosticMonitoring == null)
+        {
+            mAutomaticDiagnosticMonitoring = mPreferences.getBoolean(PREFERENCE_KEY_CHANNEL_AUTO_DIAGNOSTIC_MONITORING, true);
+        }
+
+        return mAutomaticDiagnosticMonitoring;
+    }
+
+    /**
+     * Sets the enabled state for automatic diagnostic monitoring.
+     * @param enabled true to turn on monitoring.
+     */
+    public void setAutomaticDiagnosticMonitoring(boolean enabled)
+    {
+        mAutomaticDiagnosticMonitoring = enabled;
+        mPreferences.putBoolean(PREFERENCE_KEY_CHANNEL_AUTO_DIAGNOSTIC_MONITORING, enabled);
+        notifyPreferenceUpdated();
+    }
 }