From ebe8789bd77037563064bf7c9be36dc8c2448907 Mon Sep 17 00:00:00 2001 From: idawda Date: Wed, 21 Aug 2024 11:13:40 +0530 Subject: [PATCH 1/9] NR-304574: Rate limit the IAST replay requests --- .../httpclient/IASTDataTransferRequestProcessor.java | 7 ++++++- .../com/newrelic/agent/security/util/IUtilConstants.java | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java index 4dcb65480..f1071d4ba 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java @@ -4,6 +4,7 @@ import com.newrelic.agent.security.AgentInfo; import com.newrelic.agent.security.instrumentator.utils.INRSettingsKey; import com.newrelic.agent.security.intcodeagent.filelogging.FileLoggerThreadPool; +import com.newrelic.agent.security.util.IUtilConstants; import com.newrelic.api.agent.security.utils.logging.LogLevel; import com.newrelic.agent.security.intcodeagent.models.IASTDataTransferRequest; import com.newrelic.agent.security.intcodeagent.websocket.JsonConverter; @@ -78,6 +79,9 @@ private void task() { int currentFetchThreshold = NewRelic.getAgent().getConfig() .getValue(SECURITY_POLICY_VULNERABILITY_SCAN_IAST_SCAN_PROBING_THRESHOLD, 300); + int iastFetchInterval = Math.min(Math.max(NewRelic.getAgent().getConfig().getValue(IUtilConstants.IAST_LOAD_INTERVAL, 5), 5), 500); + int fetchRatio = iastFetchInterval/5; + int remainingRecordCapacityRest = RestRequestThreadPool.getInstance().getQueue().remainingCapacity(); int currentRecordBacklogRest = RestRequestThreadPool.getInstance().getQueue().size(); int remainingRecordCapacityGrpc = GrpcClientRequestReplayHelper.getInstance().getRequestQueue().remainingCapacity(); @@ -90,8 +94,9 @@ private void task() { if(!AgentUsageMetric.isRASPProcessingActive()){ batchSize /= 2; } + batchSize /= fetchRatio; - if (batchSize > 100 && remainingRecordCapacity > batchSize) { + if (batchSize > 100/fetchRatio && remainingRecordCapacity > batchSize) { request = new IASTDataTransferRequest(NewRelicSecurity.getAgent().getAgentUUID()); if (AgentConfig.getInstance().getConfig().getCustomerInfo() != null) { request.setAppAccountId(AgentConfig.getInstance().getConfig().getCustomerInfo().getAccountId()); diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java index fd126a910..5aec9dcc5 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java @@ -12,6 +12,7 @@ public interface IUtilConstants { String SCAN_TIME_SCHEDULE = "security.scan_schedule.schedule"; String SCAN_TIME_DURATION = "security.scan_schedule.duration"; String SCAN_TIME_COLLECT_SAMPLES = "security.scan_schedule.always_sample_traces"; + String IAST_LOAD_INTERVAL = "security.scan_controllers.iast_load_interval"; String SKIP_IAST_SCAN = "security.skip_iast_scan"; String SKIP_IAST_SCAN_API = SKIP_IAST_SCAN + ".api"; From 7b6e9c878f95af472d6fe73a40058dc5426ba5c2 Mon Sep 17 00:00:00 2001 From: idawda Date: Fri, 23 Aug 2024 13:43:40 +0530 Subject: [PATCH 2/9] NR-304574: Rate limit the IAST replay requests the default value is 3600 replay req. per min --- .../IASTDataTransferRequestProcessor.java | 21 ++++++++++++------- .../agent/security/util/IUtilConstants.java | 2 +- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java index f1071d4ba..511ca2282 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java @@ -3,6 +3,7 @@ import com.newrelic.agent.security.AgentConfig; import com.newrelic.agent.security.AgentInfo; import com.newrelic.agent.security.instrumentator.utils.INRSettingsKey; +import com.newrelic.agent.security.intcodeagent.exceptions.RestrictionModeException; import com.newrelic.agent.security.intcodeagent.filelogging.FileLoggerThreadPool; import com.newrelic.agent.security.util.IUtilConstants; import com.newrelic.api.agent.security.utils.logging.LogLevel; @@ -29,8 +30,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import static com.newrelic.agent.security.instrumentator.utils.INRSettingsKey.SECURITY_POLICY_VULNERABILITY_SCAN_IAST_SCAN_PROBING_THRESHOLD; - public class IASTDataTransferRequestProcessor { private static final FileLoggerThreadPool logger = FileLoggerThreadPool.getInstance(); public static final String UNABLE_TO_SEND_IAST_DATA_REQUEST_DUE_TO_ERROR_S_S = "Unable to send IAST data request due to error: %s : %s"; @@ -48,6 +47,8 @@ public class IASTDataTransferRequestProcessor { private final AtomicLong lastFuzzCCTimestamp = new AtomicLong(); + private int currentFetchThresholdPerMin = 3600; + private void task() { IASTDataTransferRequest request = null; try { @@ -76,11 +77,12 @@ private void task() { return; } - int currentFetchThreshold = NewRelic.getAgent().getConfig() - .getValue(SECURITY_POLICY_VULNERABILITY_SCAN_IAST_SCAN_PROBING_THRESHOLD, 300); + int currentFetchThreshold = currentFetchThresholdPerMin/12; + if (currentFetchThreshold <= 0){ + return; + } - int iastFetchInterval = Math.min(Math.max(NewRelic.getAgent().getConfig().getValue(IUtilConstants.IAST_LOAD_INTERVAL, 5), 5), 500); - int fetchRatio = iastFetchInterval/5; + int fetchRatio = 300/currentFetchThreshold; int remainingRecordCapacityRest = RestRequestThreadPool.getInstance().getQueue().remainingCapacity(); int currentRecordBacklogRest = RestRequestThreadPool.getInstance().getQueue().size(); @@ -94,7 +96,6 @@ private void task() { if(!AgentUsageMetric.isRASPProcessingActive()){ batchSize /= 2; } - batchSize /= fetchRatio; if (batchSize > 100/fetchRatio && remainingRecordCapacity > batchSize) { request = new IASTDataTransferRequest(NewRelicSecurity.getAgent().getAgentUUID()); @@ -168,6 +169,12 @@ public void startDataRequestSchedule(long delay, TimeUnit timeUnit){ if(initialDelay < 0){ initialDelay = 0; } + // IAST Scan Rate per minute with range [12, 3600]; default 3600 replay requests will be replayed per minute + try { + currentFetchThresholdPerMin = Math.min(Math.max(NewRelic.getAgent().getConfig().getValue(IUtilConstants.SCAN_REQUEST_RATE_LIMIT, 3600), 12), 3600); + } catch (Exception e) { + logger.log(LogLevel.WARNING, String.format("Error while reading Configuration security.scan_request_rate_limit : %s, Using default value %s replay request per min.", e.getMessage(), currentFetchThresholdPerMin), e, this.getClass().getName()); + } logger.log(LogLevel.INFO, String.format("IAST data pull request is scheduled at %s, after delay of %s seconds", AgentConfig.getInstance().getAgentMode().getScanSchedule().getDataCollectionTime(), initialDelay), IASTDataTransferRequestProcessor.class.getName()); future = executorService.scheduleWithFixedDelay(this::task, initialDelay, delay, timeUnit); } catch (Throwable ignored){} diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java index 0c62b114d..4f5c1ac48 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java @@ -12,7 +12,7 @@ public interface IUtilConstants { String SCAN_TIME_SCHEDULE = "security.scan_schedule.schedule"; String SCAN_TIME_DURATION = "security.scan_schedule.duration"; String SCAN_TIME_COLLECT_SAMPLES = "security.scan_schedule.always_sample_traces"; - String IAST_LOAD_INTERVAL = "security.scan_controllers.iast_load_interval"; + String SCAN_REQUEST_RATE_LIMIT = "security.scan_request_rate_limit"; String SKIP_IAST_SCAN = "security.exclude_from_iast_scan"; String SKIP_IAST_SCAN_API = SKIP_IAST_SCAN + ".api"; From cfd9a05968d7bc3345135f08649a47e19a6797a7 Mon Sep 17 00:00:00 2001 From: idawda Date: Wed, 28 Aug 2024 19:20:16 +0530 Subject: [PATCH 3/9] NR-306872: Add test-identifier, process start time to websocket headers for CI/CD Support --- .../java/com/newrelic/agent/security/AgentConfig.java | 8 ++++++++ .../agent/security/intcodeagent/websocket/WSClient.java | 5 +++++ .../com/newrelic/agent/security/util/IUtilConstants.java | 1 + 3 files changed, 14 insertions(+) diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/AgentConfig.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/AgentConfig.java index 84bda35e3..3625f5861 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/AgentConfig.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/AgentConfig.java @@ -50,6 +50,8 @@ public class AgentConfig { private Map noticeErrorCustomParams = new HashMap<>(); + private String iastTestIdentifier; + private AgentConfig(){ } @@ -68,6 +70,8 @@ public void instantiate(){ logger = FileLoggerThreadPool.getInstance(); // Set required LogLevel logLevel = applyRequiredLogLevel(); + + iastTestIdentifier = NewRelic.getAgent().getConfig().getValue(IUtilConstants.IAST_TEST_IDENTIFIER); } private static final class InstanceHolder { @@ -224,4 +228,8 @@ public void setNRSecurityEnabled(boolean NRSecurityEnabled) { public String getSecurityHome() { return NR_CSEC_HOME; } + + public String getIastTestIdentifier() { + return iastTestIdentifier; + } } diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/websocket/WSClient.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/websocket/WSClient.java index 338fb464d..bdf9e8b22 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/websocket/WSClient.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/websocket/WSClient.java @@ -30,6 +30,7 @@ import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.management.ManagementFactory; import java.net.*; import java.nio.file.Files; import java.nio.file.Paths; @@ -152,6 +153,10 @@ private WSClient() throws URISyntaxException { this.addHeader("NR-CSEC-JSON-VERSION", AgentInfo.getInstance().getBuildInfo().getJsonVersion()); this.addHeader("NR-ACCOUNT-ID", AgentConfig.getInstance().getConfig().getCustomerInfo().getAccountId()); this.addHeader("NR-CSEC-IAST-DATA-TRANSFER-MODE", "PULL"); + if (AgentConfig.getInstance().getIastTestIdentifier() != null) { + this.addHeader("NR-CSEC-IAST-TEST-IDENTIFIER", AgentConfig.getInstance().getIastTestIdentifier()); + this.addHeader("NR-CSEC-PROCESS-START-TIME", String.valueOf(ManagementFactory.getRuntimeMXBean().getStartTime())); + } Proxy proxy = proxyManager(); if(proxy != null) { this.setProxy(proxy); diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java index ebe2c6f28..0045c853d 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java @@ -24,6 +24,7 @@ public interface IUtilConstants { String NR_SECURITY_ENABLED = "security.enabled"; String NR_SECURITY_HOME_APP = "security.is_home_app"; + String IAST_TEST_IDENTIFIER = "security.iast_test_identifier"; String NR_SECURITY_CA_BUNDLE_PATH = "ca_bundle_path"; String NR_CSEC_DEBUG_LOGFILE_SIZE = "NR_CSEC_DEBUG_LOGFILE_SIZE"; From 1a98cb0dd151cab0b5362f35f7cc7fe820b038bb Mon Sep 17 00:00:00 2001 From: idawda Date: Thu, 29 Aug 2024 10:18:05 +0530 Subject: [PATCH 4/9] roundup the value for currentFetchThreshold --- .../httpclient/IASTDataTransferRequestProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java index 511ca2282..a1641ac4b 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java @@ -77,7 +77,7 @@ private void task() { return; } - int currentFetchThreshold = currentFetchThresholdPerMin/12; + int currentFetchThreshold = Math.round((float) currentFetchThresholdPerMin/12); if (currentFetchThreshold <= 0){ return; } From 794311744c98853695919f0e02048c11c4e48a66 Mon Sep 17 00:00:00 2001 From: idawda Date: Fri, 30 Aug 2024 10:36:02 +0530 Subject: [PATCH 5/9] remove dependency of start-time header on test-identifier header --- .../agent/security/intcodeagent/websocket/WSClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/websocket/WSClient.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/websocket/WSClient.java index bdf9e8b22..da32c5c62 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/websocket/WSClient.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/websocket/WSClient.java @@ -155,8 +155,8 @@ private WSClient() throws URISyntaxException { this.addHeader("NR-CSEC-IAST-DATA-TRANSFER-MODE", "PULL"); if (AgentConfig.getInstance().getIastTestIdentifier() != null) { this.addHeader("NR-CSEC-IAST-TEST-IDENTIFIER", AgentConfig.getInstance().getIastTestIdentifier()); - this.addHeader("NR-CSEC-PROCESS-START-TIME", String.valueOf(ManagementFactory.getRuntimeMXBean().getStartTime())); } + this.addHeader("NR-CSEC-PROCESS-START-TIME", String.valueOf(ManagementFactory.getRuntimeMXBean().getStartTime())); Proxy proxy = proxyManager(); if(proxy != null) { this.setProxy(proxy); From 721ed8e12c5777f3ddb8792e6619dc9cf8516acf Mon Sep 17 00:00:00 2001 From: lovesh-ap Date: Mon, 2 Sep 2024 13:32:46 +0530 Subject: [PATCH 6/9] agent version bump 1.4.2 json version bump to 1.2.7 add procStartTime & scanStartTime to HealthCheck --- gradle.properties | 4 ++-- .../IASTDataTransferRequestProcessor.java | 10 ++++++++ .../models/javaagent/JAHealthCheck.java | 24 +++++++++++++++++++ .../agent/security/util/IUtilConstants.java | 2 +- 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/gradle.properties b/gradle.properties index 9cf70671d..79ca433d4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ # The agent version. -agentVersion=1.4.1 -jsonVersion=1.2.6 +agentVersion=1.4.2 +jsonVersion=1.2.7 # Updated exposed NR APM API version. nrAPIVersion=8.12.0 diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java index a1641ac4b..5c253beec 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java @@ -49,6 +49,8 @@ public class IASTDataTransferRequestProcessor { private int currentFetchThresholdPerMin = 3600; + private long scanStartEpochMilli = 0; + private void task() { IASTDataTransferRequest request = null; try { @@ -67,6 +69,10 @@ private void task() { } } long currentTimestamp = Instant.now().toEpochMilli(); + if(scanStartEpochMilli <= 0){ + AgentInfo.getInstance().getJaHealthCheck().setScanStartTime(currentTimestamp); + scanStartEpochMilli = currentTimestamp; + } // Sleep if under cooldown long cooldownSleepTime = cooldownTillTimestamp.get() - currentTimestamp; if(cooldownSleepTime > 0) { @@ -197,4 +203,8 @@ public void setCooldownTillTimestamp(long timestamp) { public void setLastFuzzCCTimestamp(long timestamp) { lastFuzzCCTimestamp.set(timestamp); } + + public long getScanStartEpochMilli() { + return scanStartEpochMilli; + } } diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/JAHealthCheck.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/JAHealthCheck.java index 7eb7f024b..57fb764ae 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/JAHealthCheck.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/JAHealthCheck.java @@ -5,6 +5,7 @@ import com.newrelic.api.agent.security.utils.logging.LogLevel; import com.newrelic.agent.security.intcodeagent.websocket.JsonConverter; +import java.lang.management.ManagementFactory; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -18,6 +19,10 @@ public class JAHealthCheck extends AgentBasicInfo { // private Set protectedDB; + private long procStartTime; + + private long scanStartTime; + private AtomicInteger invokedHookCount; private IdentifierEnvs kind; @@ -45,6 +50,7 @@ public JAHealthCheck(String applicationUUID) { this.serviceStatus = new HashMap<>(); this.eventStats = new EventStats(); this.setKind(AgentInfo.getInstance().getApplicationInfo().getIdentifier().getKind()); + this.procStartTime = ManagementFactory.getRuntimeMXBean().getStartTime(); logger.log(LogLevel.INFO, String.format(HC_CREATED, JsonConverter.toJSON(this)), JAHealthCheck.class.getName()); } @@ -59,6 +65,8 @@ public JAHealthCheck(JAHealthCheck jaHealthCheck) { this.schedulerRuns = new SchedulerRuns(jaHealthCheck.schedulerRuns); this.invokedHookCount = new AtomicInteger(jaHealthCheck.invokedHookCount.get()); this.webSocketConnectionStats = new WebSocketConnectionStats(jaHealthCheck.webSocketConnectionStats); + this.procStartTime = jaHealthCheck.getProcStartTime(); + this.scanStartTime = jaHealthCheck.getScanStartTime(); logger.log(LogLevel.INFO, String.format(HC_CREATED, JsonConverter.toJSON(this)), JAHealthCheck.class.getName()); } @@ -133,6 +141,22 @@ public void setSchedulerRuns(SchedulerRuns schedulerRuns) { this.schedulerRuns = schedulerRuns; } + public long getProcStartTime() { + return procStartTime; + } + + public void setProcStartTime(long procStartTime) { + this.procStartTime = procStartTime; + } + + public long getScanStartTime() { + return scanStartTime; + } + + public void setScanStartTime(long scanStartTime) { + this.scanStartTime = scanStartTime; + } + public void reset(){ this.setInvokedHookCount(0); this.stats.clear(); diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java index 0589266cd..d04d53cc7 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java @@ -12,7 +12,7 @@ public interface IUtilConstants { String SCAN_TIME_SCHEDULE = "security.scan_schedule.schedule"; String SCAN_TIME_DURATION = "security.scan_schedule.duration"; String SCAN_TIME_COLLECT_SAMPLES = "security.scan_schedule.always_sample_traces"; - String SCAN_REQUEST_RATE_LIMIT = "security.scan_request_rate_limit"; + String SCAN_REQUEST_RATE_LIMIT = "security.scan_controllers.iast_scan_request_rate_limit"; String SKIP_IAST_SCAN = "security.exclude_from_iast_scan"; String SKIP_IAST_SCAN_API = SKIP_IAST_SCAN + ".api"; From 5ffd62ded99a3dac571f3c47b185bfb32c02c5ae Mon Sep 17 00:00:00 2001 From: lovesh-ap Date: Mon, 2 Sep 2024 18:15:04 +0530 Subject: [PATCH 7/9] set scan start time on every NR IAST trigger --- .../src/main/java/com/newrelic/api/agent/security/Agent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java b/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java index 3aa31e43d..444b9bb0b 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java @@ -279,7 +279,7 @@ private void startSecurityServices() { } else { IASTDataTransferRequestProcessor.getInstance().stopDataRequestSchedule(true); } - + AgentInfo.getInstance().getJaHealthCheck().setScanStartTime(IASTDataTransferRequestProcessor.getInstance().getScanStartEpochMilli()); } @Override From 57be1b64184ab2c34a2280867820100492a0d123 Mon Sep 17 00:00:00 2001 From: lovesh-ap Date: Tue, 3 Sep 2024 12:47:52 +0530 Subject: [PATCH 8/9] New Requirements Enhancing Health Checks with State Information --- .../IASTDataTransferRequestProcessor.java | 14 ++--- .../ControlCommandProcessor.java | 4 ++ .../ControlCommandProcessorThreadPool.java | 9 +++ .../models/javaagent/JAHealthCheck.java | 63 +++++++++++++++++-- .../newrelic/api/agent/security/Agent.java | 9 ++- 5 files changed, 86 insertions(+), 13 deletions(-) diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java index 5c253beec..b6ae89e8d 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java @@ -3,7 +3,6 @@ import com.newrelic.agent.security.AgentConfig; import com.newrelic.agent.security.AgentInfo; import com.newrelic.agent.security.instrumentator.utils.INRSettingsKey; -import com.newrelic.agent.security.intcodeagent.exceptions.RestrictionModeException; import com.newrelic.agent.security.intcodeagent.filelogging.FileLoggerThreadPool; import com.newrelic.agent.security.util.IUtilConstants; import com.newrelic.api.agent.security.utils.logging.LogLevel; @@ -49,7 +48,7 @@ public class IASTDataTransferRequestProcessor { private int currentFetchThresholdPerMin = 3600; - private long scanStartEpochMilli = 0; + private long controlCommandRequestedAtEpochMilli = 0; private void task() { IASTDataTransferRequest request = null; @@ -69,9 +68,10 @@ private void task() { } } long currentTimestamp = Instant.now().toEpochMilli(); - if(scanStartEpochMilli <= 0){ - AgentInfo.getInstance().getJaHealthCheck().setScanStartTime(currentTimestamp); - scanStartEpochMilli = currentTimestamp; + if(controlCommandRequestedAtEpochMilli <= 0){ + AgentInfo.getInstance().getJaHealthCheck().setControlCommandRequestedAt(currentTimestamp); + controlCommandRequestedAtEpochMilli = currentTimestamp; + AgentInfo.getInstance().getJaHealthCheck().setScanActive(true); } // Sleep if under cooldown long cooldownSleepTime = cooldownTillTimestamp.get() - currentTimestamp; @@ -204,7 +204,7 @@ public void setLastFuzzCCTimestamp(long timestamp) { lastFuzzCCTimestamp.set(timestamp); } - public long getScanStartEpochMilli() { - return scanStartEpochMilli; + public long getControlCommandRequestedAtEpochMilli() { + return controlCommandRequestedAtEpochMilli; } } diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/controlcommand/ControlCommandProcessor.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/controlcommand/ControlCommandProcessor.java index 6679724f0..033913645 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/controlcommand/ControlCommandProcessor.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/controlcommand/ControlCommandProcessor.java @@ -177,6 +177,10 @@ public void run() { iastReplayRequestMsgReceiveTime = Instant.now(); IASTDataTransferRequestProcessor.getInstance().setLastFuzzCCTimestamp(Instant.now().toEpochMilli()); RestRequestProcessor.processControlCommand(controlCommand); + if(ControlCommandProcessorThreadPool.getInstance().getScanStartTime() <= 0) { + ControlCommandProcessorThreadPool.getInstance().setScanStartTime(Instant.now().toEpochMilli()); + AgentInfo.getInstance().getJaHealthCheck().setScanStartTime(ControlCommandProcessorThreadPool.getInstance().getScanStartTime()); + } break; case IntCodeControlCommand.STARTUP_WELCOME_MSG: diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/controlcommand/ControlCommandProcessorThreadPool.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/controlcommand/ControlCommandProcessorThreadPool.java index badd1c22a..0b432f3d9 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/controlcommand/ControlCommandProcessorThreadPool.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/controlcommand/ControlCommandProcessorThreadPool.java @@ -27,6 +27,8 @@ public class ControlCommandProcessorThreadPool { private final boolean allowCoreThreadTimeOut = false; private static Object mutex = new Object(); + private long scanStartTime = 0; + public ThreadPoolExecutor getExecutor() { return executor; } @@ -157,4 +159,11 @@ public void shutDownThreadPoolExecutor() { } } + public long getScanStartTime() { + return scanStartTime; + } + + public void setScanStartTime(long scanStartTime) { + this.scanStartTime = scanStartTime; + } } diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/JAHealthCheck.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/JAHealthCheck.java index 57fb764ae..235b63e6a 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/JAHealthCheck.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/JAHealthCheck.java @@ -1,11 +1,13 @@ package com.newrelic.agent.security.intcodeagent.models.javaagent; +import com.newrelic.agent.security.AgentConfig; import com.newrelic.agent.security.AgentInfo; import com.newrelic.agent.security.intcodeagent.filelogging.FileLoggerThreadPool; import com.newrelic.api.agent.security.utils.logging.LogLevel; import com.newrelic.agent.security.intcodeagent.websocket.JsonConverter; import java.lang.management.ManagementFactory; +import java.time.Instant; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -15,14 +17,20 @@ public class JAHealthCheck extends AgentBasicInfo { private static final FileLoggerThreadPool logger = FileLoggerThreadPool.getInstance(); private static final String HC_CREATED = "Created Health Check: %s"; -// private String protectedServer; - -// private Set protectedDB; - private long procStartTime; + private long controlCommandRequestedAt; + private long scanStartTime; + private long trafficStartedAt; + + private final long csecActivationTime; + + private final long iastActivationTime; + + private Boolean scanActive = false; + private AtomicInteger invokedHookCount; private IdentifierEnvs kind; @@ -51,6 +59,16 @@ public JAHealthCheck(String applicationUUID) { this.eventStats = new EventStats(); this.setKind(AgentInfo.getInstance().getApplicationInfo().getIdentifier().getKind()); this.procStartTime = ManagementFactory.getRuntimeMXBean().getStartTime(); + if(AgentConfig.getInstance().getAgentMode().getScanSchedule().getNextScanTime() != null) { + this.csecActivationTime = AgentConfig.getInstance().getAgentMode().getScanSchedule().getNextScanTime().getTime(); + } else { + this.csecActivationTime = Instant.now().toEpochMilli(); + } + if(AgentConfig.getInstance().getAgentMode().getScanSchedule().getDataCollectionTime() != null) { + this.iastActivationTime = AgentConfig.getInstance().getAgentMode().getScanSchedule().getDataCollectionTime().getTime(); + } else { + this.iastActivationTime = Instant.now().toEpochMilli(); + } logger.log(LogLevel.INFO, String.format(HC_CREATED, JsonConverter.toJSON(this)), JAHealthCheck.class.getName()); } @@ -66,7 +84,12 @@ public JAHealthCheck(JAHealthCheck jaHealthCheck) { this.invokedHookCount = new AtomicInteger(jaHealthCheck.invokedHookCount.get()); this.webSocketConnectionStats = new WebSocketConnectionStats(jaHealthCheck.webSocketConnectionStats); this.procStartTime = jaHealthCheck.getProcStartTime(); + this.controlCommandRequestedAt = jaHealthCheck.getControlCommandRequestedAt(); this.scanStartTime = jaHealthCheck.getScanStartTime(); + this.trafficStartedAt = jaHealthCheck.getTrafficStartedAt(); + this.csecActivationTime = jaHealthCheck.getCsecActivationTime(); + this.iastActivationTime = jaHealthCheck.getIastActivationTime(); + this.scanActive = jaHealthCheck.getScanActive(); logger.log(LogLevel.INFO, String.format(HC_CREATED, JsonConverter.toJSON(this)), JAHealthCheck.class.getName()); } @@ -149,6 +172,14 @@ public void setProcStartTime(long procStartTime) { this.procStartTime = procStartTime; } + public long getControlCommandRequestedAt() { + return controlCommandRequestedAt; + } + + public void setControlCommandRequestedAt(long controlCommandRequestedAt) { + this.controlCommandRequestedAt = controlCommandRequestedAt; + } + public long getScanStartTime() { return scanStartTime; } @@ -157,6 +188,30 @@ public void setScanStartTime(long scanStartTime) { this.scanStartTime = scanStartTime; } + public long getTrafficStartedAt() { + return trafficStartedAt; + } + + public void setTrafficStartedAt(long trafficStartedAt) { + this.trafficStartedAt = trafficStartedAt; + } + + public long getCsecActivationTime() { + return csecActivationTime; + } + + public Boolean getScanActive() { + return scanActive; + } + + public void setScanActive(Boolean scanActive) { + this.scanActive = scanActive; + } + + public long getIastActivationTime() { + return iastActivationTime; + } + public void reset(){ this.setInvokedHookCount(0); this.stats.clear(); diff --git a/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java b/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java index 444b9bb0b..9ce8a5f02 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java @@ -15,7 +15,6 @@ import com.newrelic.agent.security.intcodeagent.exceptions.RestrictionModeException; import com.newrelic.agent.security.intcodeagent.filelogging.FileLoggerThreadPool; import com.newrelic.agent.security.intcodeagent.filelogging.LogFileHelper; -import com.newrelic.agent.security.intcodeagent.models.collectorconfig.AgentMode; import com.newrelic.agent.security.intcodeagent.models.javaagent.*; import com.newrelic.agent.security.intcodeagent.utils.*; import com.newrelic.api.agent.security.instrumentation.helpers.*; @@ -62,6 +61,7 @@ public class Agent implements SecurityAgent { public static final String DROPPING_EVENT_AS_IT_WAS_GENERATED_BY_K_2_INTERNAL_API_CALL = "Dropping event as it was generated by agent internal API call : "; private static final AtomicBoolean firstEventProcessed = new AtomicBoolean(false); + private long trafficStartedAt = 0; public static final String ERROR_WHILE_GENERATING_TRACE_ID_FOR_CATEGORY_S = "Error while generating trace id for category : %s"; public static final String SKIPPING_THE_API_S_AS_IT_IS_PART_OF_THE_SKIP_SCAN_LIST = "Skipping the API %s as it is part of the skip scan list"; public static final String INVALID_CRON_EXPRESSION_PROVIDED_FOR_IAST_RESTRICTED_MODE = "Invalid cron expression provided for IAST Mode"; @@ -261,6 +261,7 @@ private void startSecurityServices() { ); WSReconnectionST.getInstance().submitNewTaskSchedule(0); EventSendPool.getInstance(); + ControlCommandProcessorThreadPool.getInstance(); logger.logInit( LogLevel.INFO, String.format(STARTED_MODULE_LOG, AgentServices.EventWritePool.name()), @@ -279,7 +280,8 @@ private void startSecurityServices() { } else { IASTDataTransferRequestProcessor.getInstance().stopDataRequestSchedule(true); } - AgentInfo.getInstance().getJaHealthCheck().setScanStartTime(IASTDataTransferRequestProcessor.getInstance().getScanStartEpochMilli()); + AgentInfo.getInstance().getJaHealthCheck().setControlCommandRequestedAt(IASTDataTransferRequestProcessor.getInstance().getControlCommandRequestedAtEpochMilli()); + AgentInfo.getInstance().getJaHealthCheck().setScanStartTime(ControlCommandProcessorThreadPool.getInstance().getScanStartTime()); } @Override @@ -344,6 +346,7 @@ private void deactivateSecurityServices(){ */ // InstrumentationUtils.shutdownLogic(); IASTDataTransferRequestProcessor.getInstance().stopDataRequestSchedule(true); + info.getJaHealthCheck().setScanActive(false); if(!config.getAgentMode().getScanSchedule().isCollectSamples()) { AgentInfo.getInstance().setAgentActive(false); HealthCheckScheduleThread.getInstance().cancelTask(true); @@ -468,6 +471,8 @@ public void registerOperation(AbstractOperation operation) { String.format(EVENT_ZERO_PROCESSED, securityMetaData.getRequest()), this.getClass().getName()); firstEventProcessed.set(true); + trafficStartedAt = Instant.now().toEpochMilli(); + AgentInfo.getInstance().getJaHealthCheck().setTrafficStartedAt(trafficStartedAt); } } } From 30801b7bf3a4370fed04c0774d27ea6f8c6dbede Mon Sep 17 00:00:00 2001 From: lovesh-ap Date: Tue, 3 Sep 2024 17:55:30 +0530 Subject: [PATCH 9/9] rename fields according to NR-308822 --- .../IASTDataTransferRequestProcessor.java | 2 +- .../models/javaagent/JAHealthCheck.java | 36 +++++++++---------- .../newrelic/api/agent/security/Agent.java | 4 +-- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java index b6ae89e8d..d2221a522 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/httpclient/IASTDataTransferRequestProcessor.java @@ -69,7 +69,7 @@ private void task() { } long currentTimestamp = Instant.now().toEpochMilli(); if(controlCommandRequestedAtEpochMilli <= 0){ - AgentInfo.getInstance().getJaHealthCheck().setControlCommandRequestedAt(currentTimestamp); + AgentInfo.getInstance().getJaHealthCheck().setControlCommandRequestedTime(currentTimestamp); controlCommandRequestedAtEpochMilli = currentTimestamp; AgentInfo.getInstance().getJaHealthCheck().setScanActive(true); } diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/JAHealthCheck.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/JAHealthCheck.java index 235b63e6a..cd926be52 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/JAHealthCheck.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/models/javaagent/JAHealthCheck.java @@ -19,15 +19,15 @@ public class JAHealthCheck extends AgentBasicInfo { private long procStartTime; - private long controlCommandRequestedAt; + private long controlCommandRequestedTime; private long scanStartTime; - private long trafficStartedAt; + private long trafficStartedTime; private final long csecActivationTime; - private final long iastActivationTime; + private final long iastDataRequestTime; private Boolean scanActive = false; @@ -65,9 +65,9 @@ public JAHealthCheck(String applicationUUID) { this.csecActivationTime = Instant.now().toEpochMilli(); } if(AgentConfig.getInstance().getAgentMode().getScanSchedule().getDataCollectionTime() != null) { - this.iastActivationTime = AgentConfig.getInstance().getAgentMode().getScanSchedule().getDataCollectionTime().getTime(); + this.iastDataRequestTime = AgentConfig.getInstance().getAgentMode().getScanSchedule().getDataCollectionTime().getTime(); } else { - this.iastActivationTime = Instant.now().toEpochMilli(); + this.iastDataRequestTime = Instant.now().toEpochMilli(); } logger.log(LogLevel.INFO, String.format(HC_CREATED, JsonConverter.toJSON(this)), JAHealthCheck.class.getName()); } @@ -84,11 +84,11 @@ public JAHealthCheck(JAHealthCheck jaHealthCheck) { this.invokedHookCount = new AtomicInteger(jaHealthCheck.invokedHookCount.get()); this.webSocketConnectionStats = new WebSocketConnectionStats(jaHealthCheck.webSocketConnectionStats); this.procStartTime = jaHealthCheck.getProcStartTime(); - this.controlCommandRequestedAt = jaHealthCheck.getControlCommandRequestedAt(); + this.controlCommandRequestedTime = jaHealthCheck.getControlCommandRequestedTime(); this.scanStartTime = jaHealthCheck.getScanStartTime(); - this.trafficStartedAt = jaHealthCheck.getTrafficStartedAt(); + this.trafficStartedTime = jaHealthCheck.getTrafficStartedTime(); this.csecActivationTime = jaHealthCheck.getCsecActivationTime(); - this.iastActivationTime = jaHealthCheck.getIastActivationTime(); + this.iastDataRequestTime = jaHealthCheck.getIastDataRequestTime(); this.scanActive = jaHealthCheck.getScanActive(); logger.log(LogLevel.INFO, String.format(HC_CREATED, JsonConverter.toJSON(this)), JAHealthCheck.class.getName()); } @@ -172,12 +172,12 @@ public void setProcStartTime(long procStartTime) { this.procStartTime = procStartTime; } - public long getControlCommandRequestedAt() { - return controlCommandRequestedAt; + public long getControlCommandRequestedTime() { + return controlCommandRequestedTime; } - public void setControlCommandRequestedAt(long controlCommandRequestedAt) { - this.controlCommandRequestedAt = controlCommandRequestedAt; + public void setControlCommandRequestedTime(long controlCommandRequestedTime) { + this.controlCommandRequestedTime = controlCommandRequestedTime; } public long getScanStartTime() { @@ -188,12 +188,12 @@ public void setScanStartTime(long scanStartTime) { this.scanStartTime = scanStartTime; } - public long getTrafficStartedAt() { - return trafficStartedAt; + public long getTrafficStartedTime() { + return trafficStartedTime; } - public void setTrafficStartedAt(long trafficStartedAt) { - this.trafficStartedAt = trafficStartedAt; + public void setTrafficStartedTime(long trafficStartedTime) { + this.trafficStartedTime = trafficStartedTime; } public long getCsecActivationTime() { @@ -208,8 +208,8 @@ public void setScanActive(Boolean scanActive) { this.scanActive = scanActive; } - public long getIastActivationTime() { - return iastActivationTime; + public long getIastDataRequestTime() { + return iastDataRequestTime; } public void reset(){ diff --git a/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java b/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java index 9ce8a5f02..398e61f30 100644 --- a/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java +++ b/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java @@ -280,7 +280,7 @@ private void startSecurityServices() { } else { IASTDataTransferRequestProcessor.getInstance().stopDataRequestSchedule(true); } - AgentInfo.getInstance().getJaHealthCheck().setControlCommandRequestedAt(IASTDataTransferRequestProcessor.getInstance().getControlCommandRequestedAtEpochMilli()); + AgentInfo.getInstance().getJaHealthCheck().setControlCommandRequestedTime(IASTDataTransferRequestProcessor.getInstance().getControlCommandRequestedAtEpochMilli()); AgentInfo.getInstance().getJaHealthCheck().setScanStartTime(ControlCommandProcessorThreadPool.getInstance().getScanStartTime()); } @@ -472,7 +472,7 @@ public void registerOperation(AbstractOperation operation) { this.getClass().getName()); firstEventProcessed.set(true); trafficStartedAt = Instant.now().toEpochMilli(); - AgentInfo.getInstance().getJaHealthCheck().setTrafficStartedAt(trafficStartedAt); + AgentInfo.getInstance().getJaHealthCheck().setTrafficStartedTime(trafficStartedAt); } } }