diff --git a/docs/loggers/logger_file.md b/docs/loggers/logger_file.md index e45e8955..cdfedfde 100644 --- a/docs/loggers/logger_file.md +++ b/docs/loggers/logger_file.md @@ -35,12 +35,6 @@ Options: * `compress` (boolean) > compress log file -* `compress-interval` (integer) - > checking every X seconds if new log files must be compressed - -* `compress-postcommand` (string) - > run external script after file compress step - * `mode` (string) > output format: text, jinja, json, flat-json, pcap or dnstap @@ -68,8 +62,6 @@ logfile: max-batch-size: 65536 flush-interval: 1 compress: false - compress-interval: 5 - compress-postcommand: null mode: text text-format: "" postrotate-command: null @@ -77,18 +69,41 @@ logfile: chan-buffer-size: 0 ``` -The `postrotate-command` can be used to execute a script after each file rotation. -Your script will take in argument the path file of the latest log file and then you will can do what you want on it. -If the compression is enabled then the postrotate command will be executed after that too. +### Log Compression + +When enabled, gzip log compression runs asynchronously for each completed log file. +During the rotation process, files are initially renamed with a `tocompress-` prefix, e.g., `tocompress-dnstap-1730099215373568947.log`, +indicating they’re pending compression. Once compression finishes, the file is renamed to `dnstap-1730099215373568947.log.gz`, +replacing the `tocompress-` prefix and adding the `.gz` suffix to mark completion. + +> Only one compression task runs at a time to optimize system performance, ensuring sequential compression of files. -Basic example to use the postrotate command: +To enable log compression, set `compress` to `true` in your configuration file: + +```yaml +logfile: + compress: true +``` + +### Postrotate command + +The `postrotate-command` option allows you to specify a **script** to execute after each file rotation. During the post-rotate process, files are temporarily renamed with a `toprocess-` prefix, for example, `toprocess-dnstap-1730099215373568947.log`. The script receives three arguments: +- The full path to the log file +- The directory path containing the log file +- The filename without the toprocess- prefix + +**Example Configuration** + +To specify a post-rotate command, add the following configuration: ```yaml logfile: postrotate-command: "/home/dnscollector/postrotate.sh" ``` -Script to move the log file to a specific folder +**Example Script** + +Here’s a sample script that moves the log file to a date-specific backup folder: ```bash #!/bin/bash @@ -97,9 +112,14 @@ DNSCOLLECTOR=/var/dnscollector/ BACKUP_FOLDER=$DNSCOLLECTOR/$(date +%Y-%m-%d) mkdir -p $BACKUP_FOLDER -mv $1 $BACKUP_FOLDER +# Move the log file to the backup folder, excluding the 'toprocess-' prefix from the filename +mv $1 $BACKUP_FOLDER/$3 ``` +> Note: If compression is enabled, the postrotate-command will run only after compression completes. + +### Save to PCAP files + For the `PCAP` mode, currently the DNS protocol over UDP is used to log the traffic, the following translations are done. | Origin protocol | Translated to | diff --git a/pkgconfig/loggers.go b/pkgconfig/loggers.go index a664b56b..b55f327e 100644 --- a/pkgconfig/loggers.go +++ b/pkgconfig/loggers.go @@ -68,21 +68,19 @@ type ConfigLoggers struct { ChannelBufferSize int `yaml:"chan-buffer-size" default:"0"` } `yaml:"restapi"` LogFile struct { - Enable bool `yaml:"enable" default:"false"` - FilePath string `yaml:"file-path" default:""` - MaxSize int `yaml:"max-size" default:"100"` - MaxFiles int `yaml:"max-files" default:"10"` - MaxBatchSize int `yaml:"max-batch-size" default:"65536"` - FlushInterval int `yaml:"flush-interval" default:"1"` - Compress bool `yaml:"compress" default:"false"` - CompressInterval int `yaml:"compress-interval" default:"60"` - CompressPostCommand string `yaml:"compress-postcommand" default:""` - Mode string `yaml:"mode" default:"text"` - PostRotateCommand string `yaml:"postrotate-command" default:""` - PostRotateDelete bool `yaml:"postrotate-delete-success" default:"false"` - TextFormat string `yaml:"text-format" default:""` - ChannelBufferSize int `yaml:"chan-buffer-size" default:"0"` - ExtendedSupport bool `yaml:"extended-support" default:"false"` + Enable bool `yaml:"enable" default:"false"` + FilePath string `yaml:"file-path" default:""` + MaxSize int `yaml:"max-size" default:"100"` + MaxFiles int `yaml:"max-files" default:"10"` + MaxBatchSize int `yaml:"max-batch-size" default:"65536"` + FlushInterval int `yaml:"flush-interval" default:"1"` + Compress bool `yaml:"compress" default:"false"` + Mode string `yaml:"mode" default:"text"` + PostRotateCommand string `yaml:"postrotate-command" default:""` + PostRotateDelete bool `yaml:"postrotate-delete-success" default:"false"` + TextFormat string `yaml:"text-format" default:""` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"0"` + ExtendedSupport bool `yaml:"extended-support" default:"false"` } `yaml:"logfile"` DNSTap struct { Enable bool `yaml:"enable" default:"false"` diff --git a/workers/logfile.go b/workers/logfile.go index c144dbdd..1da1ce66 100644 --- a/workers/logfile.go +++ b/workers/logfile.go @@ -56,6 +56,7 @@ type LogFile struct { fileDir, fileName, fileExt, filePrefix string textFormat []string compressQueue chan string + commandQueue chan string } func NewLogFile(config *pkgconfig.Config, logger *logger.Logger, name string) *LogFile { @@ -66,6 +67,7 @@ func NewLogFile(config *pkgconfig.Config, logger *logger.Logger, name string) *L w := &LogFile{ GenericWorker: NewGenericWorker(config, logger, name, "file", bufSize, pkgconfig.DefaultMonitor), compressQueue: make(chan string, 1), + commandQueue: make(chan string, 1), } w.ReadConfig() if err := w.OpenCurrentFile(); err != nil { @@ -76,6 +78,9 @@ func NewLogFile(config *pkgconfig.Config, logger *logger.Logger, name string) *L go w.startCompressor() w.initializeCompressionQueue() + // start post command processor + go w.startCommandProcessor() + return w } @@ -201,9 +206,13 @@ func (w *LogFile) compressFile(filename string) { // prepare dest filename baseName := filepath.Base(filename) baseName = strings.TrimPrefix(baseName, "tocompress-") + if len(w.config.Loggers.LogFile.PostRotateCommand) > 0 { + baseName = "toprocess-" + baseName + } tmpFile := filename + compressSuffix dstFile := filepath.Join(filepath.Dir(filename), baseName+compressSuffix) + // open the file fd, err := os.Open(filename) if err != nil { w.LogError("compress - failed to open file: %s", err) @@ -259,28 +268,33 @@ func (w *LogFile) compressFile(filename string) { os.Remove(tmpFile) return } + + // run post command on compressed file ? + if len(w.config.Loggers.LogFile.PostRotateCommand) > 0 { + go func() { + w.commandQueue <- dstFile + }() + } + w.LogInfo("compression terminated - %s", dstFile) } -func (w *LogFile) PostRotateCommand(filename string) { +func (w *LogFile) postRotateCommand(fullPath string) { if len(w.GetConfig().Loggers.LogFile.PostRotateCommand) > 0 { - w.LogInfo("execute postrotate command: %s", filename) - _, err := exec.Command(w.GetConfig().Loggers.LogFile.PostRotateCommand, filename).Output() + w.LogInfo("execute postrotate command: %s", fullPath) + dir := filepath.Dir(fullPath) + filename := filepath.Base(fullPath) + baseName := strings.TrimPrefix(filename, "toprocess-") + _, err := exec.Command(w.GetConfig().Loggers.LogFile.PostRotateCommand, fullPath, dir, baseName).Output() if err != nil { - w.LogError("postrotate command error: %s", err) - } else if w.GetConfig().Loggers.LogFile.PostRotateDelete { - os.Remove(filename) + w.LogError("postrotate command error - %s - %s", filename, err) + } else { + w.LogInfo("postrotate command terminated - %s", filename) } - } -} -func (w *LogFile) CompressPostRotateCommand(filename string) { - if len(w.GetConfig().Loggers.LogFile.CompressPostCommand) > 0 { - - w.LogInfo("execute compress postrotate command: %s", filename) - _, err := exec.Command(w.GetConfig().Loggers.LogFile.CompressPostCommand, filename).Output() - if err != nil { - w.LogError("compress - postcommand error: %s", err) + if w.GetConfig().Loggers.LogFile.PostRotateDelete { + w.LogInfo("postrotate command delete original file - %s", filename) + os.Remove(filename) } } } @@ -310,6 +324,10 @@ func (w *LogFile) RotateFile() error { newFilename := fmt.Sprintf("%s-%d%s", w.filePrefix, time.Now().UnixNano(), w.fileExt) if w.config.Loggers.LogFile.Compress { newFilename = fmt.Sprintf("tocompress-%s", newFilename) + } else { + if len(w.config.Loggers.LogFile.PostRotateCommand) > 0 { + newFilename = fmt.Sprintf("toprocess-%s", newFilename) + } } bfpath := filepath.Join(w.fileDir, newFilename) err := os.Rename(w.GetConfig().Loggers.LogFile.FilePath, bfpath) @@ -320,10 +338,12 @@ func (w *LogFile) RotateFile() error { // post rotate command? if w.config.Loggers.LogFile.Compress { go func() { - w.compressQueue <- bfpath // Envoi asynchrone dans le canal pour compression + w.compressQueue <- bfpath }() } else { - w.PostRotateCommand(bfpath) + go func() { + w.commandQueue <- bfpath + }() } // keep only max files @@ -449,6 +469,12 @@ func (w *LogFile) startCompressor() { } } +func (w *LogFile) startCommandProcessor() { + for filename := range w.commandQueue { + w.postRotateCommand(filename) + } +} + func (w *LogFile) StartCollect() { w.LogInfo("starting data collection") defer w.CollectDone() @@ -527,7 +553,9 @@ func (w *LogFile) StartLogging() { for { select { case <-w.OnLoggerStopped(): + // close channels close(w.compressQueue) + close(w.commandQueue) // stop timer flushTimer.Stop()