APPSERV-11 Adds Health Check Alerts to Monitoring Console

appserver/jdbc/jdbc-runtime/src/main/java/org/glassfish/jdbc/util/JdbcResourcesUtil.java

@@ -67,27 +67,20 @@
  */
 public class JdbcResourcesUtil {
 
-    private volatile static JdbcResourcesUtil jdbcResourcesUtil;
-    private static Logger _logger = LoggerFactory.getLogger(JdbcResourcesUtil.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(JdbcResourcesUtil.class);
+
+    private static final JdbcResourcesUtil instance = new JdbcResourcesUtil();
     private ConnectorRuntime runtime;
 
     private JdbcResourcesUtil() {
     }
 
     public static JdbcResourcesUtil createInstance() {
-        //stateless, no synchronization needed
-        if(jdbcResourcesUtil == null){
-            synchronized(JdbcResourcesUtil.class) {
-                if(jdbcResourcesUtil == null) {
-                    jdbcResourcesUtil = new JdbcResourcesUtil();
-                }
-            }
-        }
-        return jdbcResourcesUtil;
+        return instance;
     }
 
     private ConnectorRuntime getRuntime(){
-        if(runtime == null){
+        if (runtime == null) {
             runtime = ConnectorRuntime.getRuntime();
         }
         return runtime;
@@ -99,7 +92,7 @@ public static <T> Resource getResourceByName(Resources resources, Class<T> type,
     }
 
     public static Collection<BindableResource> getResourcesOfPool(Resources resources, String connectionPoolName) {
-        Set<BindableResource> resourcesReferringPool = new HashSet<BindableResource>();
+        Set<BindableResource> resourcesReferringPool = new HashSet<>();
         ResourcePool pool = (ResourcePool) getResourceByName(resources, ResourcePool.class, connectionPoolName);
         if (pool != null) {
             Collection<BindableResource> bindableResources = resources.getResources(BindableResource.class);
@@ -124,22 +117,22 @@ public static Collection<BindableResource> getResourcesOfPool(Resources resource
     public String getRANameofJdbcConnectionPool(JdbcConnectionPool pool) {
         String dsRAName = ConnectorConstants.JDBCDATASOURCE_RA_NAME;
 
-        Class clz = null;
+        Class<?> clz = null;
 
         if(pool.getDatasourceClassname() != null && !pool.getDatasourceClassname().isEmpty()) {
             try {
                 clz = ClassLoadingUtility.loadClass(pool.getDatasourceClassname());
             } catch (ClassNotFoundException cnfe) {
                 Object params[] = new Object[]{dsRAName, pool.getName()};
-                _logger.log(Level.WARNING, "using.default.ds", params);
+                LOGGER.log(Level.WARNING, "using.default.ds", params);
                 return dsRAName;
             }
         } else if(pool.getDriverClassname() != null && !pool.getDriverClassname().isEmpty()) {
             try {
                 clz = ClassLoadingUtility.loadClass(pool.getDriverClassname());
             } catch (ClassNotFoundException cnfe) {
                 Object params[] = new Object[]{dsRAName, pool.getName()};
-                _logger.log(Level.WARNING, "using.default.ds", params);
+                LOGGER.log(Level.WARNING, "using.default.ds", params);
                 return dsRAName;
             }
         }
@@ -175,7 +168,7 @@ public String getRANameofJdbcConnectionPool(JdbcConnectionPool pool) {
             }
         }
         Object params[] = new Object[]{dsRAName, pool.getName()};
-        _logger.log(Level.WARNING, "using.default.ds", params);
+        LOGGER.log(Level.WARNING, "using.default.ds", params);
         //default to __ds
         return dsRAName;
     }
@@ -214,18 +207,18 @@ public boolean isJdbcPoolReferredInServerInstance(PoolInfo poolInfo) {
             if ((resource.getPoolName().equalsIgnoreCase(poolInfo.getName())) &&
                     ResourcesUtil.createInstance().isReferenced(resourceInfo)
                     && ResourcesUtil.createInstance().isEnabled(resource)){
-                if (_logger.isLoggable(Level.FINE)) {
-                    _logger.fine("pool " + poolInfo + "resource " + resourceInfo
+                if (LOGGER.isLoggable(Level.FINE)) {
+                    LOGGER.fine("pool " + poolInfo + "resource " + resourceInfo
                             + " referred " + ResourcesUtil.createInstance().isReferenced(resourceInfo));
 
-                    _logger.fine("JDBC resource " + resource.getJndiName() + "refers " + poolInfo
+                    LOGGER.fine("JDBC resource " + resource.getJndiName() + "refers " + poolInfo
                             + "in this server instance and is enabled");
                 }
                 return true;
             }
         }
-        if(_logger.isLoggable(Level.FINE)) {
-            _logger.fine("No JDBC resource refers [ " + poolInfo + " ] in this server instance");
+        if(LOGGER.isLoggable(Level.FINE)) {
+            LOGGER.fine("No JDBC resource refers [ " + poolInfo + " ] in this server instance");
         }
         return false;
     }

appserver/monitoring-console/core/src/main/java/fish/payara/monitoring/alert/Alert.java

@@ -0,0 +1,246 @@
+package fish.payara.monitoring.alert;
+
+import static java.time.Instant.ofEpochMilli;
+import static java.time.LocalDateTime.ofInstant;
+import static java.time.ZoneId.systemDefault;
+
+import java.time.format.DateTimeFormatter;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import fish.payara.monitoring.model.Series;
+import fish.payara.monitoring.model.SeriesDataset;
+
+/**
+ * An {@linkplain Alert} is raised when a watched series matches the {@link Circumstance}s for {@link Level#RED} or
+ * {@link Level#AMBER} and last until the same {@link Series} and instance transitions to {@link Level#GREEN} or
+ * {@link Level#WHITE}.
+ * 
+ * @see Watch
+ * 
+ * @author Jan Bernitt
+ */
+public final class Alert implements Iterable<Alert.Frame> {
+
+    public enum Level {
+        /**
+         * Critical level
+         */
+        RED,
+        /**
+         * Elevated level to warn about
+         */
+        AMBER,
+        /**
+         * Within expected range
+         */
+        GREEN,
+        /**
+         * Not classified
+         */
+        WHITE;
+
+        public boolean isLessSevereThan(Level other) {
+            return ordinal() > other.ordinal();
+        }
+    }
+
+    public static final class Frame implements Iterable<SeriesDataset> {
+        public final Level level;
+        public final SeriesDataset cause;
+        public final long start;
+        private final List<SeriesDataset> captured;
+        long end;
+
+        public Frame(Level level, SeriesDataset cause, List<SeriesDataset> captured) {
+            this.level = level;
+            this.cause = cause;
+            this.captured = captured;
+            this.start = System.currentTimeMillis();
+        }
+
+        @Override
+        public Iterator<SeriesDataset> iterator() {
+            return captured.iterator();
+        }
+
+        public long getEnd() {
+            return end;
+        }
+    }
+
+    private static final AtomicInteger NEXT_SERIAL = new AtomicInteger();
+    /**
+     * The change count tracks state changes of all {@link Alert} instances.
+     */
+    private static final AtomicInteger CHANGE_COUNT = new AtomicInteger();
+
+    public static int getChangeCount() {
+        return CHANGE_COUNT.get();
+    }
+
+    public final int serial;
+    public final Watch initiator;
+    private final List<Frame> frames = new CopyOnWriteArrayList<>();
+    /**
+     * The current state of the alert.
+     */
+    private Level level = Level.WHITE;
+    private boolean acknowledged;
+
+    public Alert(Watch initiator) {
+        this.initiator = initiator;
+        this.serial = NEXT_SERIAL.incrementAndGet();
+    }
+
+    @Override
+    public Iterator<Frame> iterator() {
+        return frames.iterator();
+    }
+
+    public Alert addTransition(Level to, SeriesDataset cause, List<SeriesDataset> captured) {
+        assertRedOrAmberLevel(to);
+        if (!isStopped()) {
+            if (!frames.isEmpty()) {
+                Frame recent = getEndFrame();
+                assertSameSeriesAndInstance(cause, recent.cause);
+                recent.end = System.currentTimeMillis();
+                acknowledged = acknowledged && to.isLessSevereThan(recent.level);
+            } else {
+                assertMatchesWachtedSeries(cause);
+                acknowledged = false;
+            }
+            frames.add(new Frame(to, cause, captured));
+            CHANGE_COUNT.incrementAndGet();
+            level = to;
+        }
+        return this;
+    }
+
+    public boolean isStarted() {
+        return !frames.isEmpty();
+    }
+
+    public boolean isAcknowledged() {
+        return acknowledged;
+    }
+
+    public void acknowledge() {
+        if (!isAcknowledged()) {
+            acknowledged = true;
+            CHANGE_COUNT.incrementAndGet();
+        }
+    }
+
+    public boolean isStopped() {
+        return level == Level.WHITE && !frames.isEmpty();
+    }
+
+    public void stop(Level to) {
+        if (!isStopped()) {
+            assertGreenOrWhiteLevel(to);
+            this.level = to;
+            getEndFrame().end = System.currentTimeMillis();
+            CHANGE_COUNT.incrementAndGet();
+        }
+    }
+
+    public Level getLevel() {
+        return level;
+    }
+
+    public long getStartTime() {
+        return frames.isEmpty() ? -1L : frames.get(0).start;
+    }
+
+    public long getEndTime() {
+        return frames.isEmpty() ? -1L : getEndFrame().end;
+    }
+
+    public Series getSeries() {
+        return getEndFrame().cause.getSeries();
+    }
+
+    public String getInstance() {
+        return getEndFrame().cause.getInstance();
+    }
+
+    public Frame getEndFrame() {
+        return frames.get(frames.size() - 1);
+    }
+
+    @Override
+    public int hashCode() {
+        return serial;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof Alert && equalTo((Alert) obj);
+    }
+
+    public boolean equalTo(Alert other) {
+        return serial == other.serial;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder str = new StringBuilder();
+        str.append('(').append(serial).append(") ").append(initiator.name);
+        long startTime = getStartTime();
+        str.append('[');
+        if (startTime >= 0) {
+            str.append(formatTime(startTime));
+        }
+        str.append('-');
+        long endTime = getEndTime();
+        if (endTime >= 0) {
+            str.append(formatTime(endTime));
+        }
+        if (isAcknowledged()) {
+            str.append(" ACK");
+        }
+        str.append(']');
+        str.append(' ');
+        for (int i = 0; i < frames.size(); i++) {
+            if (i > 0) {
+                str.append(" => ");
+            }
+            str.append(frames.get(i).level);
+        }
+        return str.toString();
+    }
+
+    private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ISO_TIME;
+
+    private static String formatTime(long epochMillis) {
+        return TIME_FORMATTER.format(ofInstant(ofEpochMilli(epochMillis), systemDefault()));
+    }
+
+    private static void assertRedOrAmberLevel(Level to) {
+        if (to != Level.RED && to != Level.AMBER) {
+            throw new IllegalArgumentException("Alerts only transtion between RED and AMBER levels but got: " + to);
+        }
+    }
+
+    private void assertMatchesWachtedSeries(SeriesDataset cause) {
+        if (!initiator.watched.series.matches(cause.getSeries())) {
+            throw new IllegalArgumentException("Cause did not match with watched series: " + cause.getSeries());
+        }
+    }
+
+    private static void assertSameSeriesAndInstance(SeriesDataset a, SeriesDataset b) {
+        if (!b.getSeries().equalTo(a.getSeries()) || !b.getInstance().equals(a.getInstance())) {
+            throw new IllegalArgumentException(
+                    "All transitions for an alert must refer to same cause series and instance but got: " + a);
+        }
+    }
+
+    private static void assertGreenOrWhiteLevel(Level to) {
+        if (to != Level.GREEN && to != Level.WHITE) {
+            throw new IllegalArgumentException("Alerts only end on GREEN or WHITE levels but got: " + to);
+        }
+    }
+}