Merge pull request #316 from newrelic/feature/sa-dashboard-metric

Java SA dashboard data reporting via health check msgs
newrelic · Aug 12, 2024 · e693aeb · e693aeb
2 parents 44a4a17 + 8c46b8c
commit e693aeb
Show file tree

Hide file tree

Showing 25 changed files with 705 additions and 415 deletions.
diff --git a/...agent/src/main/java/com/newrelic/agent/security/instrumentator/dispatcher/Dispatcher.java b/...agent/src/main/java/com/newrelic/agent/security/instrumentator/dispatcher/Dispatcher.java
@@ -330,6 +330,7 @@ private JavaAgentEventBean processFileOperationEvent(JavaAgentEventBean eventBea
      */
     private void processReflectedXSSEvent(JavaAgentEventBean eventBean) {
         if (!NewRelic.getAgent().getConfig().getValue(INRSettingsKey.SECURITY_DETECTION_RXSS_ENABLED, true)) {
+            AgentInfo.getInstance().getJaHealthCheck().getEventStats().getDroppedDueTo().incrementRxssDetectionDeactivated();
             return;
         }
         Set<String> xssConstructs = CallbackUtils.checkForReflectedXSS(securityMetaData.getRequest(), securityMetaData.getResponse());

diff --git a/...t/src/main/java/com/newrelic/agent/security/instrumentator/dispatcher/DispatcherPool.java b/...t/src/main/java/com/newrelic/agent/security/instrumentator/dispatcher/DispatcherPool.java
@@ -83,19 +83,18 @@ public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
                         RestRequestThreadPool.getInstance().getRejectedIds().add(fuzzRequestId);
                     }
                 }
-
+                AgentInfo.getInstance().getJaHealthCheck().getEventStats().getDispatcher().incrementRejected();
                 if(dispatcher.getSecurityMetaData() != null) {
                     if(dispatcher.getSecurityMetaData().getFuzzRequestIdentifier().getK2Request()){
-                        AgentInfo.getInstance().getJaHealthCheck().getIastEventStats().incrementRejectedCount();
-                    } else {
-                        AgentInfo.getInstance().getJaHealthCheck().getRaspEventStats().incrementRejectedCount();
+                        AgentInfo.getInstance().getJaHealthCheck().getEventStats().getIastEvents().incrementRejected();
+                    }
+                    if(dispatcher.getOperation()!= null && dispatcher.getOperation().isLowSeverityHook()) {
+                        AgentInfo.getInstance().getJaHealthCheck().getEventStats().getLowSeverityEvents().incrementRejected();
                     }
                 } else if (dispatcher.getExitEventBean() != null) {
-                    AgentInfo.getInstance().getJaHealthCheck().getExitEventStats().incrementRejectedCount();
+                    AgentInfo.getInstance().getJaHealthCheck().getEventStats().getExitEvents().incrementRejected();
                 }
             }
-            AgentInfo.getInstance().getJaHealthCheck().incrementDropCount();
-            AgentInfo.getInstance().getJaHealthCheck().incrementEventRejectionCount();
 			logger.log(LogLevel.FINEST,"Event Dispatch Task " + r.toString() + " rejected from  " + e.toString(), DispatcherPool.class.getName());
         }
     }
@@ -111,13 +110,35 @@ private DispatcherPool() {
             @Override
             protected void afterExecute(Runnable r, Throwable t) {
                 try {
-                    if( t != null) {
-                        AgentInfo.getInstance().getJaHealthCheck().incrementDropCount();
-                        AgentInfo.getInstance().getJaHealthCheck().incrementEventProcessingErrorCount();
-                        incrementCount(r, IUtilConstants.ERROR);
-                    } else {
-                        AgentInfo.getInstance().getJaHealthCheck().incrementProcessedCount();
-                        incrementCount(r, IUtilConstants.PROCESSED);
+                    if (r instanceof CustomFutureTask<?> && ((CustomFutureTask<?>) r).getTask() instanceof Dispatcher) {
+                        Dispatcher dispatcher = (Dispatcher) ((CustomFutureTask<?>) r).getTask();
+                        AbstractOperation operation = dispatcher.getOperation();
+                        SecurityMetaData securityMetaData = dispatcher.getSecurityMetaData();
+                        if(t != null){
+                            AgentInfo.getInstance().getJaHealthCheck().getEventStats().getDispatcher().incrementError();
+                            if(operation != null) {
+                                if(securityMetaData != null && securityMetaData.getFuzzRequestIdentifier().getK2Request()) {
+                                    AgentInfo.getInstance().getJaHealthCheck().getEventStats().getIastEvents().incrementError();
+                                }
+                                if (operation.isLowSeverityHook()) {
+                                    AgentInfo.getInstance().getJaHealthCheck().getEventStats().getLowSeverityEvents().incrementError();
+                                }
+                            } else if (dispatcher.getExitEventBean() != null) {
+                                AgentInfo.getInstance().getJaHealthCheck().getEventStats().getExitEvents().incrementError();
+                            }
+                        } else {
+                            AgentInfo.getInstance().getJaHealthCheck().getEventStats().getDispatcher().incrementCompleted();
+                            if(operation != null) {
+                                if(securityMetaData != null && securityMetaData.getFuzzRequestIdentifier().getK2Request()) {
+                                    AgentInfo.getInstance().getJaHealthCheck().getEventStats().getIastEvents().incrementCompleted();
+                                }
+                                if (operation.isLowSeverityHook()) {
+                                    AgentInfo.getInstance().getJaHealthCheck().getEventStats().getLowSeverityEvents().incrementCompleted();
+                                }
+                            } else if (dispatcher.getExitEventBean() != null) {
+                                AgentInfo.getInstance().getJaHealthCheck().getEventStats().getExitEvents().incrementCompleted();
+                            }
+                        }
                     }
                 } catch (Throwable ignored) {
                     logger.log(LogLevel.FINEST, "Error while Dispatcher matric processing", ignored, DispatcherPool.class.getName());
@@ -145,41 +166,6 @@ public Thread newThread(Runnable r) {
         });
     }
 
-    private void incrementCount(Runnable r, String type) {
-        EventStats eventStats = null;
-        if (r instanceof CustomFutureTask<?> && ((CustomFutureTask<?>) r).getTask() instanceof Dispatcher) {
-            Dispatcher dispatcher = (Dispatcher) ((CustomFutureTask<?>) r).getTask();
-            if(dispatcher.getSecurityMetaData() != null) {
-                if(dispatcher.getSecurityMetaData().getFuzzRequestIdentifier().getK2Request()){
-                    eventStats = AgentInfo.getInstance().getJaHealthCheck().getIastEventStats();
-                } else {
-                    eventStats = AgentInfo.getInstance().getJaHealthCheck().getRaspEventStats();
-                }
-            } else if (dispatcher.getExitEventBean() != null) {
-                eventStats = AgentInfo.getInstance().getJaHealthCheck().getExitEventStats();
-            }
-        }
-        if(eventStats == null){
-            return;
-        }
-        switch (type){
-            case IUtilConstants.ERROR:
-                eventStats.incrementErrorCount();
-                break;
-            case IUtilConstants.PROCESSED:
-                eventStats.incrementProcessedCount();
-                break;
-            case IUtilConstants.SENT:
-                eventStats.incrementSentCount();
-                break;
-            case IUtilConstants.REJECTED:
-                eventStats.incrementRejectedCount();
-                break;
-            default:
-                logger.log(LogLevel.FINEST, String.format("Couldn't update event matric for task :%s and type : %s", r, type), DispatcherPool.class.getName());
-        }
-    }
-
     private static final class InstanceHolder {
         static final DispatcherPool instance = new DispatcherPool();
     }
@@ -193,15 +179,14 @@ public Set<String> getEid() {
 
 
     public void dispatchEvent(AbstractOperation operation, SecurityMetaData securityMetaData) {
-        AgentInfo.getInstance().getJaHealthCheck().incrementInvokedHookCount();
 
         if (executor.isShutdown()) {
+            AgentInfo.getInstance().getJaHealthCheck().getEventStats().getDroppedDueTo().incrementExecutorUnavailable();
             return;
         }
 
         if(!securityMetaData.getFuzzRequestIdentifier().getK2Request() && !AgentUsageMetric.isRASPProcessingActive()){
-            AgentInfo.getInstance().getJaHealthCheck().getRaspEventStats().incrementRejectedCount();
-            AgentInfo.getInstance().getJaHealthCheck().incrementEventRejectionCount();
+            AgentInfo.getInstance().getJaHealthCheck().getEventStats().getDroppedDueTo().incrementRaspProcessingDeactivated();
             return;
         }
 
@@ -233,6 +218,14 @@ public void dispatchEvent(AbstractOperation operation, SecurityMetaData security
         securityMetaData.addCustomAttribute(NR_APM_TRACE_ID, traceMetadata.getTraceId());
         securityMetaData.addCustomAttribute(NR_APM_SPAN_ID, traceMetadata.getSpanId());
         this.executor.submit(new Dispatcher(operation, new SecurityMetaData(securityMetaData)));
+        AgentInfo.getInstance().getJaHealthCheck().getEventStats().getDispatcher().incrementSubmitted();
+
+        if(securityMetaData.getFuzzRequestIdentifier().getK2Request()){
+            AgentInfo.getInstance().getJaHealthCheck().getEventStats().getIastEvents().incrementSubmitted();
+        }
+        if(operation.isLowSeverityHook()){
+            AgentInfo.getInstance().getJaHealthCheck().getEventStats().getLowSeverityEvents().incrementSubmitted();
+        }
     }
 
     public void dispatchExitEvent(ExitEventBean exitEventBean) {
@@ -246,6 +239,8 @@ public void dispatchExitEvent(ExitEventBean exitEventBean) {
         securityMetaData.addCustomAttribute(NR_APM_TRACE_ID, traceMetadata.getTraceId());
         securityMetaData.addCustomAttribute(NR_APM_SPAN_ID, traceMetadata.getSpanId());
         this.executor.submit(new Dispatcher(exitEventBean));
+        AgentInfo.getInstance().getJaHealthCheck().getEventStats().getDispatcher().incrementSubmitted();
+        AgentInfo.getInstance().getJaHealthCheck().getEventStats().getExitEvents().incrementSubmitted();
     }
 
     public static void shutDownPool() {

diff --git a/...src/main/java/com/newrelic/agent/security/instrumentator/httpclient/EventAbortPolicy.java b/...src/main/java/com/newrelic/agent/security/instrumentator/httpclient/EventAbortPolicy.java
@@ -1,5 +1,6 @@
 package com.newrelic.agent.security.instrumentator.httpclient;
 
+import com.newrelic.agent.security.AgentInfo;
 import com.newrelic.agent.security.intcodeagent.filelogging.FileLoggerThreadPool;
 import com.newrelic.api.agent.security.utils.logging.LogLevel;
 
@@ -11,6 +12,7 @@ public class EventAbortPolicy implements RejectedExecutionHandler {
 
 
     public EventAbortPolicy() {
+        AgentInfo.getInstance().getJaHealthCheck().getIastReplayRequest().incrementReplayRequestRejected();
     }
 
     public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {

diff --git a/...ain/java/com/newrelic/agent/security/instrumentator/httpclient/RestRequestThreadPool.java b/...ain/java/com/newrelic/agent/security/instrumentator/httpclient/RestRequestThreadPool.java
@@ -1,5 +1,6 @@
 package com.newrelic.agent.security.instrumentator.httpclient;
 
+import com.newrelic.agent.security.AgentInfo;
 import com.newrelic.agent.security.intcodeagent.executor.CustomFutureTask;
 import com.newrelic.agent.security.intcodeagent.executor.CustomThreadPoolExecutor;
 import com.newrelic.agent.security.intcodeagent.filelogging.FileLoggerThreadPool;

diff --git a/...java/com/newrelic/agent/security/intcodeagent/controlcommand/ControlCommandProcessor.java b/...java/com/newrelic/agent/security/intcodeagent/controlcommand/ControlCommandProcessor.java
@@ -1,6 +1,7 @@
 package com.newrelic.agent.security.intcodeagent.controlcommand;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.newrelic.agent.security.AgentInfo;
 import com.newrelic.agent.security.instrumentator.httpclient.IASTDataTransferRequestProcessor;
 import com.newrelic.agent.security.instrumentator.httpclient.RestRequestProcessor;
 import com.newrelic.agent.security.instrumentator.httpclient.RestRequestThreadPool;
@@ -168,6 +169,7 @@ public void run() {
                 }
                 break;
             case IntCodeControlCommand.FUZZ_REQUEST:
+                AgentInfo.getInstance().getJaHealthCheck().getIastReplayRequest().incrementReceivedControlCommands();
                 logger.log(LogLevel.FINER, FUZZ_REQUEST + controlCommandMessage,
                         ControlCommandProcessor.class.getName());
                 IASTDataTransferRequestProcessor.getInstance().setLastFuzzCCTimestamp(Instant.now().toEpochMilli());
@@ -216,6 +218,7 @@ public void run() {
                  * Post reconnect: reset 'reconnecting phase' in WSClient.
                  */
                 try {
+                    AgentInfo.getInstance().getJaHealthCheck().getWebSocketConnectionStats().incrementReceivedReconnectAtWill();
                     //TODO no need for draining IAST since last leg has complete ledger.
                     logger.log(LogLevel.INFO, RECEIVED_WS_RECONNECT_COMMAND_FROM_SERVER_INITIATING_SEQUENCE, this.getClass().getName());
                     if (NewRelicSecurity.getAgent().getCurrentPolicy().getVulnerabilityScan().getEnabled() &&

diff --git a/...ewrelic/agent/security/intcodeagent/controlcommand/ControlCommandProcessorThreadPool.java b/...ewrelic/agent/security/intcodeagent/controlcommand/ControlCommandProcessorThreadPool.java
@@ -27,6 +27,10 @@ public class ControlCommandProcessorThreadPool {
     private final boolean allowCoreThreadTimeOut = false;
     private static Object mutex = new Object();
 
+    public ThreadPoolExecutor getExecutor() {
+        return executor;
+    }
+
     /**
      * A handler for rejected tasks that throws a
      * {@code RejectedExecutionException}.

diff --git a/.../main/java/com/newrelic/agent/security/intcodeagent/filelogging/FileLoggerThreadPool.java b/.../main/java/com/newrelic/agent/security/intcodeagent/filelogging/FileLoggerThreadPool.java
@@ -205,4 +205,8 @@ public void setInitLoggingActive(boolean initLoggingActive) {
     public boolean isLogLevelEnabled(LogLevel logLevel) {
         return (logLevel.getLevel() >= LogWriter.defaultLogLevel);
     }
+
+    public ThreadPoolExecutor getExecutor() {
+        return executor;
+    }
 }
diff --git a/...ent/src/main/java/com/newrelic/agent/security/intcodeagent/filelogging/LogFileHelper.java b/...ent/src/main/java/com/newrelic/agent/security/intcodeagent/filelogging/LogFileHelper.java
@@ -7,6 +7,7 @@
 
 package com.newrelic.agent.security.intcodeagent.filelogging;
 
+import com.newrelic.agent.security.AgentInfo;
 import com.newrelic.agent.security.instrumentator.os.OsVariablesInstance;
 import com.newrelic.agent.security.intcodeagent.properties.K2JALogProperties;
 import com.newrelic.agent.security.util.IUtilConstants;
@@ -100,6 +101,7 @@ public static BufferedWriter dailyRollover(String fileName) throws IOException {
 
     public static void performDailyRollover(){
         try {
+            AgentInfo.getInstance().getJaHealthCheck().getSchedulerRuns().incrementDailyLogRollover();
             InitLogWriter.setWriter(dailyRollover(InitLogWriter.getFileName()));
         } catch (IOException e) {
             FileLoggerThreadPool.getInstance().setInitLoggingActive(false);

diff --git a/...main/java/com/newrelic/agent/security/intcodeagent/logging/HealthCheckScheduleThread.java b/...main/java/com/newrelic/agent/security/intcodeagent/logging/HealthCheckScheduleThread.java
@@ -7,7 +7,10 @@
 import com.newrelic.agent.security.instrumentator.os.OSVariables;
 import com.newrelic.agent.security.instrumentator.os.OsVariablesInstance;
 import com.newrelic.agent.security.instrumentator.utils.AgentUtils;
+import com.newrelic.agent.security.intcodeagent.controlcommand.ControlCommandProcessorThreadPool;
 import com.newrelic.agent.security.intcodeagent.filelogging.FileLoggerThreadPool;
+import com.newrelic.agent.security.intcodeagent.models.javaagent.ThreadPoolActiveStat;
+import com.newrelic.api.agent.security.instrumentation.helpers.GrpcClientRequestReplayHelper;
 import com.newrelic.api.agent.security.utils.logging.LogLevel;
 import com.newrelic.agent.security.intcodeagent.models.javaagent.JAHealthCheck;
 import com.newrelic.agent.security.intcodeagent.models.javaagent.ThreadPoolStats;
@@ -76,7 +79,8 @@ public void run() {
                 }
 
                 logger.log(LogLevel.INFO, String.format("Pending CCs to be processed : %s", RestRequestThreadPool.getInstance().getQueueSize()), this.getClass().getName());
-                AgentInfo.getInstance().getJaHealthCheck().setDsBackLog(RestRequestThreadPool.getInstance().getQueueSize());
+                AgentInfo.getInstance().getJaHealthCheck().getIastReplayRequest().incrementPendingControlCommandsBy(RestRequestThreadPool.getInstance().getQueueSize());
+                AgentInfo.getInstance().getJaHealthCheck().getIastReplayRequest().incrementPendingControlCommandsBy(GrpcClientRequestReplayHelper.getInstance().getRequestQueue().size());
                 AgentUtils.getInstance().addStatusLogMostRecentHCs(AgentInfo.getInstance().getJaHealthCheck().toString());
 //						channel.write(ByteBuffer.wrap(new JAHealthCheck(AgentNew.JA_HEALTH_CHECK).toString().getBytes()));
                 if (WSClient.getInstance().isOpen()) {
@@ -101,8 +105,17 @@ public void run() {
 
     private ThreadPoolStats populateThreadPoolStats() {
         ThreadPoolStats threadPoolStats = new ThreadPoolStats();
-        threadPoolStats.setDispatcherQueueSize(DispatcherPool.getInstance().getExecutor().getQueue().size());
-        threadPoolStats.setEventSendQueueSize(EventSendPool.getInstance().getExecutor().getQueue().size());
+        threadPoolStats.setDispatcher(new ThreadPoolActiveStat(DispatcherPool.getInstance().getExecutor().getActiveCount(),
+                DispatcherPool.getInstance().getExecutor().getQueue().size()));
+        threadPoolStats.setEventSender(new ThreadPoolActiveStat(EventSendPool.getInstance().getExecutor().getActiveCount(),
+                EventSendPool.getInstance().getExecutor().getQueue().size()));
+        threadPoolStats.setControlCommandProcessor(new ThreadPoolActiveStat(ControlCommandProcessorThreadPool.getInstance().getExecutor().getActiveCount(),
+                ControlCommandProcessorThreadPool.getInstance().getExecutor().getQueue().size()));
+        threadPoolStats.setIastHttpRequestProcessor(new ThreadPoolActiveStat(RestRequestThreadPool.getInstance().getExecutor().getActiveCount(),
+                RestRequestThreadPool.getInstance().getExecutor().getQueue().size()));
+        threadPoolStats.setFileLogger(new ThreadPoolActiveStat(FileLoggerThreadPool.getInstance().getExecutor().getActiveCount(),
+                FileLoggerThreadPool.getInstance().getExecutor().getQueue().size()));
+
         return threadPoolStats;
     }
 

diff --git a/...c/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/AgentBasicInfo.java b/...c/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/AgentBasicInfo.java
@@ -67,8 +67,6 @@ public class AgentBasicInfo {
     @JsonInclude
     private static String policyVersion;
 
-    private String accountId;
-
     private boolean isPolicyOverridden = AgentUtils.getInstance().isPolicyOverridden();
 
     /**