Skip to content

Commit

Permalink
file logger: no more block postrotate command
Browse files Browse the repository at this point in the history
  • Loading branch information
dmachard committed Oct 28, 2024
1 parent 9e8f442 commit 0e3c212
Show file tree
Hide file tree
Showing 3 changed files with 92 additions and 46 deletions.
48 changes: 34 additions & 14 deletions docs/loggers/logger_file.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,6 @@ Options:
* `compress` (boolean)
> compress log file
* `compress-interval` (integer)
> checking every X seconds if new log files must be compressed
* `compress-postcommand` (string)
> run external script after file compress step
* `mode` (string)
> output format: text, jinja, json, flat-json, pcap or dnstap
Expand Down Expand Up @@ -68,27 +62,48 @@ logfile:
max-batch-size: 65536
flush-interval: 1
compress: false
compress-interval: 5
compress-postcommand: null
mode: text
text-format: ""
postrotate-command: null
postrotate-delete-success: false
chan-buffer-size: 0
```
The `postrotate-command` can be used to execute a script after each file rotation.
Your script will take in argument the path file of the latest log file and then you will can do what you want on it.
If the compression is enabled then the postrotate command will be executed after that too.
### Log Compression
When enabled, gzip log compression runs asynchronously for each completed log file.
During the rotation process, files are initially renamed with a `tocompress-` prefix, e.g., `tocompress-dnstap-1730099215373568947.log`,
indicating they’re pending compression. Once compression finishes, the file is renamed to `dnstap-1730099215373568947.log.gz`,
replacing the `tocompress-` prefix and adding the `.gz` suffix to mark completion.

> Only one compression task runs at a time to optimize system performance, ensuring sequential compression of files.

Basic example to use the postrotate command:
To enable log compression, set `compress` to `true` in your configuration file:

```yaml
logfile:
compress: true
```

### Postrotate command

The `postrotate-command` option allows you to specify a **script** to execute after each file rotation. During the post-rotate process, files are temporarily renamed with a `toprocess-` prefix, for example, `toprocess-dnstap-1730099215373568947.log`. The script receives three arguments:
- The full path to the log file
- The directory path containing the log file
- The filename without the toprocess- prefix

**Example Configuration**

To specify a post-rotate command, add the following configuration:

```yaml
logfile:
postrotate-command: "/home/dnscollector/postrotate.sh"
```

Script to move the log file to a specific folder
**Example Script**

Here’s a sample script that moves the log file to a date-specific backup folder:

```bash
#!/bin/bash
Expand All @@ -97,9 +112,14 @@ DNSCOLLECTOR=/var/dnscollector/
BACKUP_FOLDER=$DNSCOLLECTOR/$(date +%Y-%m-%d)
mkdir -p $BACKUP_FOLDER
mv $1 $BACKUP_FOLDER
# Move the log file to the backup folder, excluding the 'toprocess-' prefix from the filename
mv $1 $BACKUP_FOLDER/$3
```

> Note: If compression is enabled, the postrotate-command will run only after compression completes.

### Save to PCAP files

For the `PCAP` mode, currently the DNS protocol over UDP is used to log the traffic, the following translations are done.

| Origin protocol | Translated to |
Expand Down
28 changes: 13 additions & 15 deletions pkgconfig/loggers.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,21 +68,19 @@ type ConfigLoggers struct {
ChannelBufferSize int `yaml:"chan-buffer-size" default:"0"`
} `yaml:"restapi"`
LogFile struct {
Enable bool `yaml:"enable" default:"false"`
FilePath string `yaml:"file-path" default:""`
MaxSize int `yaml:"max-size" default:"100"`
MaxFiles int `yaml:"max-files" default:"10"`
MaxBatchSize int `yaml:"max-batch-size" default:"65536"`
FlushInterval int `yaml:"flush-interval" default:"1"`
Compress bool `yaml:"compress" default:"false"`
CompressInterval int `yaml:"compress-interval" default:"60"`
CompressPostCommand string `yaml:"compress-postcommand" default:""`
Mode string `yaml:"mode" default:"text"`
PostRotateCommand string `yaml:"postrotate-command" default:""`
PostRotateDelete bool `yaml:"postrotate-delete-success" default:"false"`
TextFormat string `yaml:"text-format" default:""`
ChannelBufferSize int `yaml:"chan-buffer-size" default:"0"`
ExtendedSupport bool `yaml:"extended-support" default:"false"`
Enable bool `yaml:"enable" default:"false"`
FilePath string `yaml:"file-path" default:""`
MaxSize int `yaml:"max-size" default:"100"`
MaxFiles int `yaml:"max-files" default:"10"`
MaxBatchSize int `yaml:"max-batch-size" default:"65536"`
FlushInterval int `yaml:"flush-interval" default:"1"`
Compress bool `yaml:"compress" default:"false"`
Mode string `yaml:"mode" default:"text"`
PostRotateCommand string `yaml:"postrotate-command" default:""`
PostRotateDelete bool `yaml:"postrotate-delete-success" default:"false"`
TextFormat string `yaml:"text-format" default:""`
ChannelBufferSize int `yaml:"chan-buffer-size" default:"0"`
ExtendedSupport bool `yaml:"extended-support" default:"false"`
} `yaml:"logfile"`
DNSTap struct {
Enable bool `yaml:"enable" default:"false"`
Expand Down
62 changes: 45 additions & 17 deletions workers/logfile.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ type LogFile struct {
fileDir, fileName, fileExt, filePrefix string
textFormat []string
compressQueue chan string
commandQueue chan string
}

func NewLogFile(config *pkgconfig.Config, logger *logger.Logger, name string) *LogFile {
Expand All @@ -66,6 +67,7 @@ func NewLogFile(config *pkgconfig.Config, logger *logger.Logger, name string) *L
w := &LogFile{
GenericWorker: NewGenericWorker(config, logger, name, "file", bufSize, pkgconfig.DefaultMonitor),
compressQueue: make(chan string, 1),
commandQueue: make(chan string, 1),
}
w.ReadConfig()
if err := w.OpenCurrentFile(); err != nil {
Expand All @@ -76,6 +78,9 @@ func NewLogFile(config *pkgconfig.Config, logger *logger.Logger, name string) *L
go w.startCompressor()
w.initializeCompressionQueue()

// start post command processor
go w.startCommandProcessor()

return w
}

Expand Down Expand Up @@ -201,9 +206,13 @@ func (w *LogFile) compressFile(filename string) {
// prepare dest filename
baseName := filepath.Base(filename)
baseName = strings.TrimPrefix(baseName, "tocompress-")
if len(w.config.Loggers.LogFile.PostRotateCommand) > 0 {
baseName = "toprocess-" + baseName
}
tmpFile := filename + compressSuffix
dstFile := filepath.Join(filepath.Dir(filename), baseName+compressSuffix)

// open the file
fd, err := os.Open(filename)
if err != nil {
w.LogError("compress - failed to open file: %s", err)
Expand Down Expand Up @@ -259,28 +268,33 @@ func (w *LogFile) compressFile(filename string) {
os.Remove(tmpFile)
return
}

// run post command on compressed file ?
if len(w.config.Loggers.LogFile.PostRotateCommand) > 0 {
go func() {
w.commandQueue <- dstFile
}()
}

w.LogInfo("compression terminated - %s", dstFile)
}

func (w *LogFile) PostRotateCommand(filename string) {
func (w *LogFile) postRotateCommand(fullPath string) {
if len(w.GetConfig().Loggers.LogFile.PostRotateCommand) > 0 {
w.LogInfo("execute postrotate command: %s", filename)
_, err := exec.Command(w.GetConfig().Loggers.LogFile.PostRotateCommand, filename).Output()
w.LogInfo("execute postrotate command: %s", fullPath)
dir := filepath.Dir(fullPath)
filename := filepath.Base(fullPath)
baseName := strings.TrimPrefix(filename, "toprocess-")
_, err := exec.Command(w.GetConfig().Loggers.LogFile.PostRotateCommand, fullPath, dir, baseName).Output()
if err != nil {
w.LogError("postrotate command error: %s", err)
} else if w.GetConfig().Loggers.LogFile.PostRotateDelete {
os.Remove(filename)
w.LogError("postrotate command error - %s - %s", filename, err)
} else {
w.LogInfo("postrotate command terminated - %s", filename)
}
}
}

func (w *LogFile) CompressPostRotateCommand(filename string) {
if len(w.GetConfig().Loggers.LogFile.CompressPostCommand) > 0 {

w.LogInfo("execute compress postrotate command: %s", filename)
_, err := exec.Command(w.GetConfig().Loggers.LogFile.CompressPostCommand, filename).Output()
if err != nil {
w.LogError("compress - postcommand error: %s", err)
if w.GetConfig().Loggers.LogFile.PostRotateDelete {
w.LogInfo("postrotate command delete original file - %s", filename)
os.Remove(filename)
}
}
}
Expand Down Expand Up @@ -310,6 +324,10 @@ func (w *LogFile) RotateFile() error {
newFilename := fmt.Sprintf("%s-%d%s", w.filePrefix, time.Now().UnixNano(), w.fileExt)
if w.config.Loggers.LogFile.Compress {
newFilename = fmt.Sprintf("tocompress-%s", newFilename)
} else {

Check failure on line 327 in workers/logfile.go

View workflow job for this annotation

GitHub Actions / linter

elseif: can replace 'else {if cond {}}' with 'else if cond {}' (gocritic)
if len(w.config.Loggers.LogFile.PostRotateCommand) > 0 {
newFilename = fmt.Sprintf("toprocess-%s", newFilename)
}
}
bfpath := filepath.Join(w.fileDir, newFilename)
err := os.Rename(w.GetConfig().Loggers.LogFile.FilePath, bfpath)
Expand All @@ -320,10 +338,12 @@ func (w *LogFile) RotateFile() error {
// post rotate command?
if w.config.Loggers.LogFile.Compress {
go func() {
w.compressQueue <- bfpath // Envoi asynchrone dans le canal pour compression
w.compressQueue <- bfpath
}()
} else {
w.PostRotateCommand(bfpath)
go func() {
w.commandQueue <- bfpath
}()
}

// keep only max files
Expand Down Expand Up @@ -449,6 +469,12 @@ func (w *LogFile) startCompressor() {
}
}

func (w *LogFile) startCommandProcessor() {
for filename := range w.commandQueue {
w.postRotateCommand(filename)
}
}

func (w *LogFile) StartCollect() {
w.LogInfo("starting data collection")
defer w.CollectDone()
Expand Down Expand Up @@ -527,7 +553,9 @@ func (w *LogFile) StartLogging() {
for {
select {
case <-w.OnLoggerStopped():
// close channels
close(w.compressQueue)
close(w.commandQueue)

// stop timer
flushTimer.Stop()
Expand Down

0 comments on commit 0e3c212

Please sign in to comment.