Logger arch update (#113)

* support for std out logging * add feature for Daily log rollover * Support logger config : log_file_count and log_limit_in_kbytes * Remove NR_CSEC_DEBUG_LOGFILE_SIZE and NR_CSEC_DEBUG_LOGFILE_MAX_COUNT
newrelic · Oct 5, 2023 · b1bed4b · b1bed4b
1 parent 2bb58e9
commit b1bed4b
Show file tree

Hide file tree

Showing 10 changed files with 166 additions and 50 deletions.
diff --git a/...gent/src/main/java/com/newrelic/agent/security/instrumentator/os/OsVariablesInstance.java b/...gent/src/main/java/com/newrelic/agent/security/instrumentator/os/OsVariablesInstance.java
@@ -18,6 +18,8 @@ public class OsVariablesInstance {
     public static final String TMP = "tmp";
     public static final String SNAPSHOTS = "snapshots";
 
+    public static final String LOG_FILE_PATH = "log_file_path";
+
     private static OsVariablesInstance instance;
 
     private final static Object lock = new Object();
@@ -40,8 +42,8 @@ private OsVariablesInstance() {
 //        osVariables.setLogDirectory(Paths.get(k2root.toString(), LOGS, LANGUAGE_AGENT, AgentInfo.getInstance().getApplicationUUID()).toString());
 
 
-        if(NewRelic.getAgent().getConfig().getValue("log_file_path") != null) {
-            osVariables.setLogDirectory(Paths.get(NewRelic.getAgent().getConfig().getValue("log_file_path"), IUtilConstants.NR_SECURITY_HOME, LOGS).toString());
+        if(NewRelic.getAgent().getConfig().getValue(LOG_FILE_PATH) != null) {
+            osVariables.setLogDirectory(Paths.get(NewRelic.getAgent().getConfig().getValue(LOG_FILE_PATH), IUtilConstants.NR_SECURITY_HOME, LOGS).toString());
         } else {
             osVariables.setLogDirectory(Paths.get(AgentConfig.getInstance().getK2Home(), LOGS).toString());
         }

diff --git a/.../main/java/com/newrelic/agent/security/intcodeagent/filelogging/FileLoggerThreadPool.java b/.../main/java/com/newrelic/agent/security/intcodeagent/filelogging/FileLoggerThreadPool.java
@@ -20,16 +20,20 @@ public class FileLoggerThreadPool {
 
     private boolean isInitLoggingActive = true;
 
-    protected int maxfilesize = K2JALogProperties.maxfilesize;
+    protected final int maxfilesize;
 
-    protected int maxfiles = K2JALogProperties.maxfiles;
+    protected final int maxfiles;
+
+    protected boolean isLoggingToStdOut = false;
 
     private FileLoggerThreadPool() throws IOException {
         // load the settings
         int queueSize = 15000;
         int maxPoolSize = 1;
         int corePoolSize = 1;
         long keepAliveTime = 600;
+        maxfiles = Math.max(K2JALogProperties.maxfiles, LogFileHelper.logFileCount());
+        maxfilesize = LogFileHelper.logFileLimit()*1024;
 
         TimeUnit timeUnit = TimeUnit.SECONDS;
 
@@ -60,12 +64,8 @@ public Thread newThread(Runnable r) {
             }
         });
         try {
-            if (System.getenv().containsKey(IUtilConstants.NR_CSEC_DEBUG_LOGFILE_SIZE)) {
-                this.maxfilesize = Integer.parseInt(System.getenv().get(IUtilConstants.NR_CSEC_DEBUG_LOGFILE_SIZE));
-            }
-
-            if (System.getenv().containsKey(IUtilConstants.NR_CSEC_DEBUG_LOGFILE_MAX_COUNT)) {
-                this.maxfiles = Integer.parseInt(System.getenv().get(IUtilConstants.NR_CSEC_DEBUG_LOGFILE_MAX_COUNT));
+            if(LogFileHelper.isLoggingToStdOut()){
+                this.isLoggingToStdOut = true;
             }
         } catch (NumberFormatException e){}
 
@@ -141,7 +141,9 @@ public void logInit(LogLevel logLevel, String event, String logSourceClassName)
         if (logLevel.getLevel() == 1 || logLevel.getLevel() > InitLogWriter.defaultLogLevel) {
             return;
         }
-        executor.submit(new InitLogWriter(logLevel, event, logSourceClassName, Thread.currentThread().getName()));
+        if(!isLoggingToStdOut) {
+            executor.submit(new InitLogWriter(logLevel, event, logSourceClassName, Thread.currentThread().getName()));
+        }
         log(logLevel, event, logSourceClassName);
     }
 
@@ -150,7 +152,9 @@ public void logInit(LogLevel logLevel, String event, Throwable throwableEvent, S
         if (logLevel.getLevel() == 1 || logLevel.getLevel() > InitLogWriter.defaultLogLevel) {
             return;
         }
-        executor.submit(new InitLogWriter(logLevel, event, throwableEvent, logSourceClassName, Thread.currentThread().getName()));
+        if(!isLoggingToStdOut) {
+            executor.submit(new InitLogWriter(logLevel, event, throwableEvent, logSourceClassName, Thread.currentThread().getName()));
+        }
         log(logLevel, event, throwableEvent, logSourceClassName);
     }
 

diff --git a/...ent/src/main/java/com/newrelic/agent/security/intcodeagent/filelogging/InitLogWriter.java b/...ent/src/main/java/com/newrelic/agent/security/intcodeagent/filelogging/InitLogWriter.java
@@ -52,31 +52,37 @@ public class InitLogWriter implements Runnable {
 
     public static final String DATE_FORMAT_NOW = "yyyy-MM-dd HH:mm:ss";
 
-    private static final String fileName;
+    private static String fileName;
 
-    private static final String currentLogFileName;
-
-    private static int logFileCounter = 0;
+    private static String currentLogFileName;
 
     private static BufferedWriter writer;
 
-    private static final File currentLogFile;
+    private static File currentLogFile;
 
     private String threadName;
 
     private static OSVariables osVariables = OsVariablesInstance.getInstance().getOsVariables();
 
     static {
-        fileName = new File(osVariables.getLogDirectory(), "java-security-collector-init.log").getAbsolutePath();
-        currentLogFile = new File(fileName);
-        CommonUtils.forceMkdirs(currentLogFile.getParentFile().toPath(), "rwxrwxrwx");
-        currentLogFileName = fileName;
+        if(FileLoggerThreadPool.getInstance().isLoggingToStdOut){
+            writer = new BufferedWriter(new OutputStreamWriter(System.out));
+        } else {
+            fileName = new File(osVariables.getLogDirectory(), "java-security-collector-init.log").getAbsolutePath();
+            currentLogFile = new File(fileName);
+            CommonUtils.forceMkdirs(currentLogFile.getParentFile().toPath(), "rwxrwxrwx");
+            currentLogFileName = fileName;
+            createLogFile();
+        }
+    }
+
+    private static void createLogFile() {
         try {
             currentLogFile.setReadable(true, false);
             writer = new BufferedWriter(new FileWriter(currentLogFileName, true));
             writer.write(LOG_FILE_INITIATED_MSG);
             writer.flush();
-            maxFileSize = FileLoggerThreadPool.getInstance().maxfilesize * 1048576;
+            maxFileSize = FileLoggerThreadPool.getInstance().maxfilesize;
 
             // k2.log.handler.maxfilesize=10
             // k2.log.handler.maxfilesize.unit=MB
@@ -165,7 +171,7 @@ public void run() {
     }
 
     private static void rollover(String fileName) throws IOException {
-        if (!rolloverCheckNeeded()) {
+        if (FileLoggerThreadPool.getInstance().isLoggingToStdOut || !rolloverCheckNeeded()) {
             return;
         }
 
@@ -182,7 +188,7 @@ private static void rollover(String fileName) throws IOException {
                 } catch (IOException e) {
                 }
 
-                CommonUtils.deleteRolloverLogFiles(currentFile.getName(), FileLoggerThreadPool.getInstance().maxfiles);
+                LogFileHelper.deleteRolloverLogFiles(currentFile.getName(), FileLoggerThreadPool.getInstance().maxfiles);
             }
         } finally {
             writer = new BufferedWriter(new FileWriter(currentFile, true));
@@ -211,4 +217,7 @@ public static String getFileName() {
         return fileName;
     }
 
+    public static void setWriter(BufferedWriter writer) {
+        InitLogWriter.writer = writer;
+    }
 }
diff --git a/...ent/src/main/java/com/newrelic/agent/security/intcodeagent/filelogging/LogFileHelper.java b/...ent/src/main/java/com/newrelic/agent/security/intcodeagent/filelogging/LogFileHelper.java
@@ -0,0 +1,105 @@
+/*
+ *
+ *  * Copyright 2020 New Relic Corporation. All rights reserved.
+ *  * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package com.newrelic.agent.security.intcodeagent.filelogging;
+
+import com.newrelic.agent.security.instrumentator.os.OsVariablesInstance;
+import com.newrelic.agent.security.intcodeagent.properties.K2JALogProperties;
+import com.newrelic.api.agent.NewRelic;
+import com.newrelic.api.agent.security.schema.StringUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.comparator.LastModifiedFileComparator;
+import org.apache.commons.io.filefilter.FileFilterUtils;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ * Utility class to get the New Relic log file.
+ */
+public class LogFileHelper {
+
+    public static final String LOG_DAILY = "log_daily";
+    public static final String LOG_FILE_COUNT = "log_file_count";
+    public static final String LOG_FILE_NAME = "log_file_name";
+
+    public static final String LOG_LIMIT = "log_limit_in_kbytes";
+    public static final boolean DEFAULT_LOG_DAILY = false;
+    public static final int DEFAULT_LOG_FILE_COUNT = 1;
+    public static final String DEFAULT_LOG_FILE_NAME = "java-security-collector.log";
+
+    public static final String STDOUT = "STDOUT";
+
+    private static final String STRING_DOT = ".";
+
+    public static boolean isLoggingToStdOut() {
+        String logFileName = NewRelic.getAgent().getConfig().getValue(LogFileHelper.LOG_FILE_NAME, LogFileHelper.DEFAULT_LOG_FILE_NAME);
+        return StringUtils.equals(LogFileHelper.STDOUT, logFileName);
+    }
+
+    public static int logFileCount() {
+        return Math.max(1, NewRelic.getAgent().getConfig().getValue(LogFileHelper.LOG_FILE_COUNT, LogFileHelper.DEFAULT_LOG_FILE_COUNT));
+    }
+
+    public static int logFileLimit() {
+        int size = NewRelic.getAgent().getConfig().getValue(LogFileHelper.LOG_LIMIT, 1);
+        return size>1?size: K2JALogProperties.maxfilesize;
+    }
+
+    public static boolean isDailyRollover() {
+        return NewRelic.getAgent().getConfig().getValue(LogFileHelper.LOG_DAILY, LogFileHelper.DEFAULT_LOG_DAILY);
+    }
+
+    public static void deleteRolloverLogFiles(String fileName, int max) {
+        Collection<File> rolloverLogFiles = FileUtils.listFiles(new File(OsVariablesInstance.getInstance().getOsVariables().getLogDirectory()), FileFilterUtils.prefixFileFilter(fileName + "."), null);
+
+        if (rolloverLogFiles.size() > max) {
+            File[] sortedLogFiles = rolloverLogFiles.toArray(new File[0]);
+            Arrays.sort(sortedLogFiles, LastModifiedFileComparator.LASTMODIFIED_COMPARATOR);
+            for (int i = 0; i < sortedLogFiles.length - max; i++) {
+                FileUtils.deleteQuietly(sortedLogFiles[i]);
+            }
+        }
+    }
+
+    public static BufferedWriter dailyRollover(String fileName) throws IOException {
+        File currentFile = new File(fileName);
+        try {
+            File rolloverFile = new File(fileName + STRING_DOT + Instant.now().toEpochMilli());
+            FileUtils.moveFile(currentFile, rolloverFile);
+            deleteRolloverLogFiles(currentFile.getName(), FileLoggerThreadPool.getInstance().maxfiles);
+            currentFile.setReadable(true, false);
+            currentFile.setWritable(true, false);
+            if (!OsVariablesInstance.getInstance().getOsVariables().getWindows()) {
+                Files.setPosixFilePermissions(currentFile.toPath(), PosixFilePermissions.fromString("rw-rw-rw-"));
+            }
+        } catch (IOException e) {
+        }
+        return new BufferedWriter(new FileWriter(currentFile, true));
+    }
+
+    public static void performDailyRollover(){
+        try {
+            InitLogWriter.setWriter(dailyRollover(InitLogWriter.getFileName()));
+        } catch (IOException e) {
+            FileLoggerThreadPool.getInstance().setInitLoggingActive(false);
+        }
+        try {
+            LogWriter.setWriter(dailyRollover(LogWriter.getFileName()));
+        } catch (IOException e) {
+            FileLoggerThreadPool.getInstance().setLoggingActive(false);
+        }
+
+    }
+}
diff --git a/...y-agent/src/main/java/com/newrelic/agent/security/intcodeagent/filelogging/LogWriter.java b/...y-agent/src/main/java/com/newrelic/agent/security/intcodeagent/filelogging/LogWriter.java
@@ -48,13 +48,13 @@ public class LogWriter implements Runnable {
 
     public static final String DATE_FORMAT_NOW = "yyyy-MM-dd HH:mm:ss.SSS";
 
-    private static final String fileName;
+    private static String fileName;
 
-    private static final String currentLogFileName;
+    private static String currentLogFileName;
 
     private static BufferedWriter writer;
 
-    private static final File currentLogFile;
+    private static File currentLogFile;
 
     private String threadName;
 
@@ -68,7 +68,7 @@ private static boolean createLogFile() {
             currentLogFile.setReadable(true, false);
             writer = new BufferedWriter(new FileWriter(currentLogFileName, true));
 
-            maxFileSize = FileLoggerThreadPool.getInstance().maxfilesize * 1048576;
+            maxFileSize = FileLoggerThreadPool.getInstance().maxfilesize;
 
             if (!osVariables.getWindows()) {
                 Files.setPosixFilePermissions(currentLogFile.toPath(), PosixFilePermissions.fromString("rw-rw-rw-"));
@@ -91,10 +91,14 @@ private static boolean createLogFile() {
     }
 
     static {
-        fileName = new File(osVariables.getLogDirectory(), "java-security-collector.log").getAbsolutePath();
-        currentLogFile = new File(fileName);
-        currentLogFileName = fileName;
-        createLogFile();
+        if(FileLoggerThreadPool.getInstance().isLoggingToStdOut){
+            writer = new BufferedWriter(new OutputStreamWriter(System.out));
+        } else {
+            fileName = new File(osVariables.getLogDirectory(), "java-security-collector.log").getAbsolutePath();
+            currentLogFile = new File(fileName);
+            currentLogFileName = fileName;
+            createLogFile();
+        }
     }
 
     public LogWriter(LogLevel logLevel, String logEntry, String loggingClassName, String threadName) {
@@ -171,7 +175,7 @@ public void run() {
     }
 
     private static void rollover(String fileName) throws IOException {
-        if (!rolloverCheckNeeded()) {
+        if (FileLoggerThreadPool.getInstance().isLoggingToStdOut || !rolloverCheckNeeded()) {
             return;
         }
 
@@ -189,7 +193,7 @@ private static void rollover(String fileName) throws IOException {
                 } catch (IOException e) {
                 }
 
-                CommonUtils.deleteRolloverLogFiles(currentFile.getName(), FileLoggerThreadPool.getInstance().maxfiles);
+                LogFileHelper.deleteRolloverLogFiles(currentFile.getName(), FileLoggerThreadPool.getInstance().maxfiles);
             }
         } finally {
             writer = new BufferedWriter(new FileWriter(currentFile, true));
@@ -220,4 +224,7 @@ public static String getFileName() {
         return fileName;
     }
 
+    public static void setWriter(BufferedWriter writer) {
+        LogWriter.writer = writer;
+    }
 }
diff --git a/.../src/main/java/com/newrelic/agent/security/intcodeagent/properties/K2JALogProperties.java b/.../src/main/java/com/newrelic/agent/security/intcodeagent/properties/K2JALogProperties.java
@@ -1,6 +1,6 @@
 package com.newrelic.agent.security.intcodeagent.properties;
 
 public interface K2JALogProperties {
-    int maxfilesize = 50;
+    int maxfilesize = 50*1024;
     int maxfiles = 2;
 }
diff --git a/...nt/src/main/java/com/newrelic/agent/security/intcodeagent/schedulers/SchedulerHelper.java b/...nt/src/main/java/com/newrelic/agent/security/intcodeagent/schedulers/SchedulerHelper.java
@@ -2,7 +2,6 @@
 
 import com.newrelic.agent.security.intcodeagent.filelogging.FileLoggerThreadPool;
 import com.newrelic.agent.security.intcodeagent.filelogging.LogFileHelper;
-import com.newrelic.agent.security.intcodeagent.filelogging.LogLevel;
 import com.newrelic.agent.security.intcodeagent.logging.IAgentConstants;
 import com.newrelic.agent.security.util.IUtilConstants;
 import com.newrelic.api.agent.NewRelic;
@@ -68,7 +67,7 @@ public ScheduledFuture<?> scheduleLowSeverityFilterCleanup(Runnable command,
     }
 
     public ScheduledFuture<?> scheduleDailyLogRollover(Runnable command) {
-        logger.log(LogLevel.INFO, "Start ", SchedulerHelper.class.getName());
+
         if(LogFileHelper.isDailyRollover()) {
             int period = NewRelic.getAgent().getConfig().getValue(IUtilConstants.NR_LOG_DAILY_ROLLOVER_PERIOD, 24);
             ScheduledFuture<?> future = commonExecutor.scheduleWithFixedDelay(command, period, period, TimeUnit.HOURS);

diff --git a/...urity-agent/src/main/java/com/newrelic/agent/security/intcodeagent/utils/CommonUtils.java b/...urity-agent/src/main/java/com/newrelic/agent/security/intcodeagent/utils/CommonUtils.java
@@ -140,19 +140,6 @@ public static InputStream getResourceStreamFromAgentJar(String resourceName) {
         return null;
     }
 
-    public static void deleteRolloverLogFiles(String fileName, int max) {
-        Collection<File> rolloverLogFiles = FileUtils.listFiles(new File(OsVariablesInstance.getInstance().getOsVariables().getLogDirectory()), FileFilterUtils.prefixFileFilter(fileName + "."), null);
-
-        if (rolloverLogFiles.size() > max) {
-            File[] sortedLogFiles = rolloverLogFiles.toArray(new File[0]);
-            Arrays.sort(sortedLogFiles, LastModifiedFileComparator.LASTMODIFIED_COMPARATOR);
-            for (int i = 0; i < sortedLogFiles.length - max; i++) {
-                FileUtils.deleteQuietly(sortedLogFiles[i]);
-
-            }
-        }
-    }
-
 
     /**
      * Generate random int between range start to end. Both inclusive.

diff --git a/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java b/newrelic-security-agent/src/main/java/com/newrelic/agent/security/util/IUtilConstants.java
@@ -26,4 +26,5 @@ public interface IUtilConstants {
     String ERROR = "ERROR";
     String SENT = "SENT";
     String REJECTED = "REJECTED";
+    String NR_LOG_DAILY_ROLLOVER_PERIOD = "log.rollover.period";
 }
diff --git a/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java b/newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java
@@ -15,6 +15,7 @@
 import com.newrelic.agent.security.instrumentator.utils.INRSettingsKey;
 import com.newrelic.agent.security.intcodeagent.constants.AgentServices;
 import com.newrelic.agent.security.intcodeagent.filelogging.FileLoggerThreadPool;
+import com.newrelic.agent.security.intcodeagent.filelogging.LogFileHelper;
 import com.newrelic.agent.security.intcodeagent.filelogging.LogLevel;
 import com.newrelic.agent.security.intcodeagent.logging.HealthCheckScheduleThread;
 import com.newrelic.agent.security.intcodeagent.logging.IAgentConstants;
@@ -185,6 +186,7 @@ private void startK2Services() {
         FileCleaner.scheduleNewTask();
         SchedulerHelper.getInstance().scheduleLowSeverityFilterCleanup(LowSeverityHelper::clearLowSeverityEventFilter,
                 30 , 30, TimeUnit.MINUTES);
+        SchedulerHelper.getInstance().scheduleDailyLogRollover(LogFileHelper::performDailyRollover);
         logger.logInit(
                 LogLevel.INFO,
                 String.format(STARTED_MODULE_LOG, AgentServices.HealthCheck.name()),