From f2ae2819927c44487305f370edb1e143b047a5c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 22 Nov 2021 10:39:03 +0100 Subject: [PATCH] Remove `logging.files.suffix` option and always use datetime suffixes (#28927) ## What does this PR do? This PR contains several improvements to the output file log rotation: * Removes suffix option as discussed offline * Filenames end in extension name so users get a hint about the contents * Removes `time.Sleep` from testing code * Datetime format no longer contains hour and minutes. In case of conflict on rotation, an index is appended to the filename. ## Why is it important? Previously, log file suffixes were configurable. Users had the option to either add the count or the datetime to the end of the log file. From now on, we only allow datetime based naming. Hence the configuration option `logging.files.suffix` is removed. Example: Filebeat is writing logs to `filebeat-20211111.ndjson` actively. Then a few minutes later it gets rotated, and the new active file becomes `filebeat-20211111-1.ndjson`. This change should help with Beats not being able to rotate files on Windows. --- CHANGELOG.next.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 5 - filebeat/filebeat.reference.yml | 5 - filebeat/tests/system/filebeat.py | 2 +- filebeat/tests/system/test_harvester.py | 4 +- filebeat/tests/system/test_input.py | 6 +- filebeat/tests/system/test_load.py | 2 +- filebeat/tests/system/test_multiline.py | 8 +- filebeat/tests/system/test_registrar.py | 16 +- heartbeat/heartbeat.reference.yml | 5 - journalbeat/journalbeat.reference.yml | 5 - .../_meta/config/logging.reference.yml.tmpl | 5 - libbeat/common/file/interval_rotator.go | 192 ---------------- libbeat/common/file/interval_rotator_test.go | 154 ------------- libbeat/common/file/rotator.go | 215 ++++++++---------- libbeat/common/file/rotator_test.go | 162 +++++++------ libbeat/common/file/trigger.go | 8 +- libbeat/docs/loggingconfig.asciidoc | 9 - libbeat/logp/config.go | 20 +- libbeat/logp/core.go | 1 - libbeat/logp/core_test.go | 2 +- libbeat/outputs/fileout/config.go | 16 +- libbeat/outputs/fileout/file.go | 1 - libbeat/tests/system/base.py | 2 + libbeat/tests/system/beat/beat.py | 27 ++- libbeat/tests/system/test_base.py | 3 +- libbeat/tests/system/test_logging.py | 4 +- libbeat/tests/system/test_umask.py | 2 +- metricbeat/metricbeat.reference.yml | 5 - packetbeat/packetbeat.reference.yml | 5 - packetbeat/tests/system/packetbeat.py | 17 +- .../tests/system/test_0002_thrift_basics.py | 2 +- packetbeat/tests/system/test_0006_wsgi.py | 4 +- winlogbeat/tests/system/test_config.py | 2 +- winlogbeat/tests/system/test_wineventlog.py | 2 +- winlogbeat/winlogbeat.reference.yml | 5 - x-pack/auditbeat/auditbeat.reference.yml | 5 - .../pkg/agent/operation/monitoring.go | 6 +- .../elastic-agent/pkg/core/logger/logger.go | 4 +- .../core/monitoring/beats/beats_monitor.go | 2 +- .../pkg/core/monitoring/beats/monitoring.go | 4 +- x-pack/filebeat/filebeat.reference.yml | 5 - .../functionbeat/functionbeat.reference.yml | 5 - x-pack/heartbeat/heartbeat.reference.yml | 5 - x-pack/metricbeat/metricbeat.reference.yml | 5 - x-pack/osquerybeat/osquerybeat.reference.yml | 5 - x-pack/packetbeat/packetbeat.reference.yml | 5 - x-pack/winlogbeat/winlogbeat.reference.yml | 5 - 48 files changed, 254 insertions(+), 726 deletions(-) delete mode 100644 libbeat/common/file/interval_rotator.go delete mode 100644 libbeat/common/file/interval_rotator_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index db654552a58..66ba4760319 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -27,6 +27,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - add_process_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620] - add_docker_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620] - Index template's default_fields setting is only populated with ECS fields. {pull}28596[28596] {issue}28215[28215] +- Remove options `logging.files.suffix` and default to datetime endings. {pull}28927[28927] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 3d1435314b4..1068dbd8082 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -1443,11 +1443,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. auditbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index dc7250e402c..69c6e855c10 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -2355,11 +2355,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. filebeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/filebeat/tests/system/filebeat.py b/filebeat/tests/system/filebeat.py index 92df15911bb..9f1f7905d16 100644 --- a/filebeat/tests/system/filebeat.py +++ b/filebeat/tests/system/filebeat.py @@ -36,7 +36,7 @@ def access_registry(self, name=None, data_path=None): return Registry(data_path, name) def log_access(self, file=None): - file = file if file else self.beat_name + ".log" + file = file if file else self.beat_name + "-" + self.today + ".ndjson" return LogState(os.path.join(self.working_dir, file)) def has_registry(self, name=None, data_path=None): diff --git a/filebeat/tests/system/test_harvester.py b/filebeat/tests/system/test_harvester.py index f56a51d0fe0..ecbb23fab77 100644 --- a/filebeat/tests/system/test_harvester.py +++ b/filebeat/tests/system/test_harvester.py @@ -493,11 +493,11 @@ def test_boms(self, fb_encoding, py_encoding, bom): filebeat = self.start_beat(output=fb_encoding + ".log") self.wait_until( - lambda: self.output_has(lines=1, output_file="output/" + fb_encoding), + lambda: self.output_has(lines=1, output_file="output/" + fb_encoding + "-" + self.today + ".ndjson"), max_timeout=10) # Verify that output does not contain bom - output = self.read_output_json(output_file="output/" + fb_encoding) + output = self.read_output_json(output_file="output/" + fb_encoding + "-" + self.today + ".ndjson") assert output[0]["message"] == message filebeat.kill_and_wait() diff --git a/filebeat/tests/system/test_input.py b/filebeat/tests/system/test_input.py index f9dbd138a2e..a4215587735 100644 --- a/filebeat/tests/system/test_input.py +++ b/filebeat/tests/system/test_input.py @@ -633,10 +633,10 @@ def test_restart_recursive_glob(self): with open(testfile_path, 'a') as testfile: testfile.write("entry2\n") - filebeat = self.start_beat(output="filebeat2.log") + filebeat = self.start_beat() self.wait_until( - lambda: self.output_has_message("entry2"), + lambda: self.output_has_message("entry2", output_file="output/filebeat-"+self.today+"-1.ndjson"), max_timeout=10, name="output contains 'entry2'") @@ -783,7 +783,7 @@ def test_inode_marker_based_identity_tracking_to_path_based(self): proc = self.start_beat() # on startup output is rotated - self.wait_until(lambda: self.output_has(lines=1, output_file="output/filebeat.1")) + self.wait_until(lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson")) self.wait_until(lambda: self.output_has(lines=1)) proc.check_kill_and_wait() diff --git a/filebeat/tests/system/test_load.py b/filebeat/tests/system/test_load.py index 6a35a6ce250..b48b83e2659 100644 --- a/filebeat/tests/system/test_load.py +++ b/filebeat/tests/system/test_load.py @@ -72,7 +72,7 @@ def test_no_missing_events(self): entry_list = [] - with open(self.working_dir + "/output/filebeat") as f: + with open(self.working_dir + "/output/filebeat-" + self.today + ".ndjson") as f: for line in f: content = json.loads(line) v = int(content["message"]) diff --git a/filebeat/tests/system/test_multiline.py b/filebeat/tests/system/test_multiline.py index ee5384ef5ef..31d4f0258ca 100644 --- a/filebeat/tests/system/test_multiline.py +++ b/filebeat/tests/system/test_multiline.py @@ -148,11 +148,11 @@ def test_max_lines(self): # Checks line 3 is sent assert True == self.log_contains( - "MetaDataMappingService.java:388", "output/filebeat") + "MetaDataMappingService.java:388", "output/filebeat-" + self.today + ".ndjson") # Checks line 4 is not sent anymore assert False == self.log_contains( - "InternalClusterService.java:388", "output/filebeat") + "InternalClusterService.java:388", "output/filebeat-" + self.today + ".ndjson") # Check that output file has the same number of lines as the log file assert 20 == len(output) @@ -231,10 +231,10 @@ def test_max_bytes(self): output = self.read_output() # Check that first 60 chars are sent - assert True == self.log_contains("cluster.metadata", "output/filebeat") + assert True == self.log_contains("cluster.metadata", "output/filebeat-" + self.today + ".ndjson") # Checks that chars afterwards are not sent - assert False == self.log_contains("Zach", "output/filebeat") + assert False == self.log_contains("Zach", "output/filebeat-" + self.today + ".ndjson") # Check that output file has the same number of lines as the log file assert 20 == len(output) diff --git a/filebeat/tests/system/test_registrar.py b/filebeat/tests/system/test_registrar.py index 02d07a32446..bb55ef547b3 100644 --- a/filebeat/tests/system/test_registrar.py +++ b/filebeat/tests/system/test_registrar.py @@ -486,11 +486,11 @@ def test_restart_continue(self): # Output file was rotated self.wait_until( - lambda: self.output_has(lines=1, output_file="output/filebeat.1"), + lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + ".ndjson"), max_timeout=10) self.wait_until( - lambda: self.output_has(lines=1), + lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson"), max_timeout=10) filebeat.check_kill_and_wait() @@ -505,7 +505,7 @@ def test_restart_continue(self): # should never have been detected assert len(data) == 1 - output = self.read_output() + output = self.read_output(output_file="output/filebeat-" + self.today + "-1.ndjson") # Check that output file has the same number of lines as the log file assert len(output) == 1 @@ -592,15 +592,15 @@ def test_rotating_file_with_restart(self): with open(testfile_path, 'w') as testfile: testfile.write("entry3\n") - filebeat = self.start_beat(output="filebeat2.log") + filebeat = self.start_beat() # Output file was rotated self.wait_until( - lambda: self.output_has(lines=2, output_file="output/filebeat.1"), + lambda: self.output_has(lines=2), max_timeout=10) self.wait_until( - lambda: self.output_has(lines=1), + lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson"), max_timeout=10) filebeat.check_kill_and_wait() @@ -951,8 +951,8 @@ def test_restart_state(self): clean_inactive="3s", ) - filebeat = self.start_beat(output="filebeat2.log") - logs = self.log_access("filebeat2.log") + filebeat = self.start_beat() + logs = self.log_access() # Write additional file for name in restart_files: diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 1eb37188303..6bac78d08c2 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1589,11 +1589,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. heartbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Heartbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 5058ed19cce..2170f2fcec4 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -1386,11 +1386,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. journalbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Journalbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/libbeat/_meta/config/logging.reference.yml.tmpl b/libbeat/_meta/config/logging.reference.yml.tmpl index 9a1e91a0af8..f4ca435be01 100644 --- a/libbeat/_meta/config/logging.reference.yml.tmpl +++ b/libbeat/_meta/config/logging.reference.yml.tmpl @@ -67,8 +67,3 @@ logging.files: # Rotate existing logs on startup rather than appending to the existing # file. Defaults to true. # rotateonstartup: true - - # Rotated files are either suffixed with a number e.g. {{.BeatName}}.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count diff --git a/libbeat/common/file/interval_rotator.go b/libbeat/common/file/interval_rotator.go deleted file mode 100644 index e5105beb0a7..00000000000 --- a/libbeat/common/file/interval_rotator.go +++ /dev/null @@ -1,192 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package file - -import ( - "fmt" - "os" - "path/filepath" - "sort" - "strconv" - "time" - - "github.com/pkg/errors" -) - -type intervalRotator struct { - log Logger - interval time.Duration - lastRotate time.Time - filename string - fileFormat string - clock clock - weekly bool - arbitrary bool -} - -func newIntervalRotator(log Logger, interval time.Duration, filename string) rotater { - ir := &intervalRotator{ - filename: filename, - log: log, - interval: (interval / time.Second) * time.Second, // drop fractional seconds - clock: realClock{}, - } - ir.initialize() - return ir -} - -func (r *intervalRotator) initialize() { - switch r.interval { - case time.Second: - r.fileFormat = "2006-01-02-15-04-05" - case time.Minute: - r.fileFormat = "2006-01-02-15-04" - case time.Hour: - r.fileFormat = "2006-01-02-15" - case 24 * time.Hour: // calendar day - r.fileFormat = "2006-01-02" - case 7 * 24 * time.Hour: // calendar week - r.fileFormat = "" - r.weekly = true - case 30 * 24 * time.Hour: // calendar month - r.fileFormat = "2006-01" - case 365 * 24 * time.Hour: // calendar year - r.fileFormat = "2006" - default: - r.arbitrary = true - r.fileFormat = "2006-01-02-15-04-05" - } - - fi, err := os.Stat(r.filename) - if err != nil { - if r.log != nil { - r.log.Debugw("Not attempting to find last rotated time, configured logs dir cannot be opened: %v", err) - } - return - } - r.lastRotate = fi.ModTime() -} - -func (r *intervalRotator) ActiveFile() string { - return r.filename -} - -func (r *intervalRotator) LogPrefix(filename string, modTime time.Time) string { - var t time.Time - if r.lastRotate.IsZero() { - t = modTime - } else { - t = r.lastRotate - } - - if r.weekly { - y, w := t.ISOWeek() - return fmt.Sprintf("%s-%04d-%02d-", filename, y, w) - } - if r.arbitrary { - intervalNumber := t.Unix() / (int64(r.interval) / int64(time.Second)) - intervalStart := time.Unix(0, intervalNumber*int64(r.interval)) - return fmt.Sprintf("%s-%s-", filename, intervalStart.Format(r.fileFormat)) - } - return fmt.Sprintf("%s-%s-", filename, t.Format(r.fileFormat)) -} - -func (r *intervalRotator) RotatedFiles() []string { - files, err := filepath.Glob(r.filename + "*") - if err != nil { - if r.log != nil { - r.log.Debugw("failed to list existing logs: %+v", err) - } - } - r.SortIntervalLogs(files) - return files -} - -func (r *intervalRotator) Rotate(reason rotateReason, t time.Time) error { - fi, err := os.Stat(r.ActiveFile()) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return errors.Wrap(err, "failed to rotate backups") - } - - logPrefix := r.LogPrefix(r.ActiveFile(), fi.ModTime()) - files, err := filepath.Glob(logPrefix + "*") - if err != nil { - return errors.Wrap(err, "failed to list logs during rotation") - } - - var targetFilename string - if len(files) == 0 { - targetFilename = logPrefix + "1" - } else { - r.SortIntervalLogs(files) - lastLogIndex, _, err := IntervalLogIndex(files[len(files)-1]) - if err != nil { - return errors.Wrap(err, "failed to locate last log index during rotation") - } - targetFilename = logPrefix + strconv.Itoa(int(lastLogIndex)+1) - } - - if err := os.Rename(r.ActiveFile(), targetFilename); err != nil { - return errors.Wrap(err, "failed to rotate backups") - } - - if r.log != nil { - r.log.Debugw("Rotating file", "filename", r.ActiveFile(), "reason", reason) - } - - r.lastRotate = t - return nil -} - -func (r *intervalRotator) SortIntervalLogs(strings []string) { - sort.Slice( - strings, - func(i, j int) bool { - return OrderIntervalLogs(strings[i]) < OrderIntervalLogs(strings[j]) - }, - ) -} - -// OrderIntervalLogs, when given a log filename in the form [prefix]-[formattedDate]-n -// returns the filename after zero-padding the trailing n so that foo-[date]-2 sorts -// before foo-[date]-10. -func OrderIntervalLogs(filename string) string { - index, i, err := IntervalLogIndex(filename) - if err == nil { - return filename[:i] + fmt.Sprintf("%020d", index) - } - - return "" -} - -// IntervalLogIndex returns n as int given a log filename in the form [prefix]-[formattedDate]-n -func IntervalLogIndex(filename string) (uint64, int, error) { - i := len(filename) - 1 - for ; i >= 0; i-- { - if '0' > filename[i] || filename[i] > '9' { - break - } - } - i++ - - s64 := filename[i:] - u64, err := strconv.ParseUint(s64, 10, 64) - return u64, i, err -} diff --git a/libbeat/common/file/interval_rotator_test.go b/libbeat/common/file/interval_rotator_test.go deleted file mode 100644 index 729d8c713b7..00000000000 --- a/libbeat/common/file/interval_rotator_test.go +++ /dev/null @@ -1,154 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package file - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestSecondRotator(t *testing.T) { - a := newMockIntervalRotator(time.Second) - - clock := &testClock{time.Date(2018, 12, 31, 0, 0, 1, 100, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-00-00-01-", a.LogPrefix("foo", time.Now())) -} - -func TestMinuteRotator(t *testing.T) { - a := newMockIntervalRotator(time.Minute) - - clock := &testClock{time.Date(2018, 12, 31, 0, 1, 1, 0, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-00-01-", a.LogPrefix("foo", time.Now())) -} - -func TestHourlyRotator(t *testing.T) { - a := newMockIntervalRotator(time.Hour) - - clock := &testClock{time.Date(2018, 12, 31, 1, 0, 1, 0, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-01-", a.LogPrefix("foo", time.Now())) -} - -func TestDailyRotator(t *testing.T) { - a := newMockIntervalRotator(24 * time.Hour) - - clock := &testClock{time.Date(2018, 12, 31, 0, 0, 0, 0, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-", a.LogPrefix("foo", time.Now())) -} - -func TestWeeklyRotator(t *testing.T) { - a := newMockIntervalRotator(7 * 24 * time.Hour) - - // Monday, 2018-Dec-31 - clock := &testClock{time.Date(2018, 12, 31, 0, 0, 0, 0, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2019-01-", a.LogPrefix("foo", time.Now())) - - // Monday, 2019-Jan-7 - clock.time = clock.time.Add(7 * 24 * time.Hour) - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2019-02-", a.LogPrefix("foo", time.Now())) -} - -func TestMonthlyRotator(t *testing.T) { - a := newMockIntervalRotator(30 * 24 * time.Hour) - - clock := &testClock{time.Date(2018, 12, 1, 0, 0, 0, 0, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(30 * 24 * time.Hour) - assert.Equal(t, "foo-2018-12-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(24 * time.Hour) - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2019-01-", a.LogPrefix("foo", time.Now())) -} - -func TestYearlyRotator(t *testing.T) { - a := newMockIntervalRotator(365 * 24 * time.Hour) - - clock := &testClock{time.Date(2018, 12, 31, 0, 0, 0, 0, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(23 * time.Hour) - assert.Equal(t, "foo-2018-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Hour) - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2019-", a.LogPrefix("foo", time.Now())) -} - -func TestArbitraryIntervalRotator(t *testing.T) { - a := newMockIntervalRotator(3 * time.Second) - - // Monday, 2018-Dec-31 - clock := &testClock{time.Date(2018, 12, 31, 0, 0, 1, 0, time.Local)} - a.clock = clock - assert.Equal(t, "foo-2018-12-30-00-00-00-", a.LogPrefix("foo", time.Date(2018, 12, 30, 0, 0, 0, 0, time.Local))) - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-00-00-00-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - assert.Equal(t, "foo-2018-12-31-00-00-00-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-00-00-03-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - assert.Equal(t, "foo-2018-12-31-00-00-03-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - assert.Equal(t, "foo-2018-12-31-00-00-03-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-00-00-06-", a.LogPrefix("foo", time.Now())) -} - -func TestIntervalIsTruncatedToSeconds(t *testing.T) { - a := newMockIntervalRotator(2345 * time.Millisecond) - assert.Equal(t, 2*time.Second, a.interval) -} - -type testClock struct { - time time.Time -} - -func (t testClock) Now() time.Time { - return t.time -} - -func newMockIntervalRotator(interval time.Duration) *intervalRotator { - r := newIntervalRotator(nil, interval, "foo").(*intervalRotator) - return r -} diff --git a/libbeat/common/file/rotator.go b/libbeat/common/file/rotator.go index 633f94232ef..e32e317dbfb 100644 --- a/libbeat/common/file/rotator.go +++ b/libbeat/common/file/rotator.go @@ -29,22 +29,13 @@ import ( "github.com/pkg/errors" ) -type SuffixType uint32 - const ( // MaxBackupsLimit is the upper bound on the number of backup files. Any values // greater will result in an error. MaxBackupsLimit = 1024 - - SuffixCount SuffixType = iota + 1 - SuffixDate + DateFormat = "20060102" ) -var suffixes = map[string]SuffixType{ - "count": SuffixCount, - "date": SuffixDate, -} - // rotater is the interface responsible for rotating and finding files. type rotater interface { // ActiveFile returns the path to the file that is actively written. @@ -69,9 +60,9 @@ type Rotator struct { interval time.Duration permissions os.FileMode log Logger // Optional Logger (may be nil). - suffix SuffixType rotateOnStartup bool redirectStderr bool + clock clock file *os.File mutex sync.Mutex @@ -85,14 +76,6 @@ type Logger interface { // RotatorOption is a configuration option for Rotator. type RotatorOption func(r *Rotator) -// Interval sets the time interval for log rotation in addition to log -// rotation by size. The default is 0 for disabled. -func Suffix(s SuffixType) RotatorOption { - return func(r *Rotator) { - r.suffix = s - } -} - // MaxSizeBytes configures the maximum number of bytes that a file should // contain before being rotated. The default is 10 MiB. func MaxSizeBytes(n uint) RotatorOption { @@ -150,6 +133,12 @@ func RedirectStderr(redirect bool) RotatorOption { } } +func WithClock(clock clock) RotatorOption { + return func(r *Rotator) { + r.clock = clock + } +} + // NewFileRotator returns a new Rotator. func NewFileRotator(filename string, options ...RotatorOption) (*Rotator, error) { r := &Rotator{ @@ -158,7 +147,7 @@ func NewFileRotator(filename string, options ...RotatorOption) (*Rotator, error) permissions: 0600, interval: 0, rotateOnStartup: true, - suffix: SuffixCount, + clock: &realClock{}, } for _, opt := range options { @@ -179,14 +168,14 @@ func NewFileRotator(filename string, options ...RotatorOption) (*Rotator, error) return nil, errors.New("the minimum time interval for log rotation is 1 second") } - r.rot = newRotater(r.log, r.suffix, filename, r.maxBackups, r.interval) + r.rot = newDateRotater(r.log, filename, r.clock) shouldRotateOnStart := r.rotateOnStartup if _, err := os.Stat(r.rot.ActiveFile()); os.IsNotExist(err) { shouldRotateOnStart = false } - r.triggers = newTriggers(shouldRotateOnStart, r.interval, r.maxSizeBytes) + r.triggers = newTriggers(shouldRotateOnStart, r.interval, r.maxSizeBytes, r.clock) if r.log != nil { r.log.Debugw("Initialized file rotator", @@ -194,7 +183,6 @@ func NewFileRotator(filename string, options ...RotatorOption) (*Rotator, error) "max_size_bytes", r.maxSizeBytes, "max_backups", r.maxBackups, "permissions", r.permissions, - "suffix", r.suffix, ) } @@ -292,7 +280,7 @@ func (r *Rotator) openFile() error { } func (r *Rotator) rotate(reason rotateReason) error { - return r.rotateWithTime(reason, time.Now()) + return r.rotateWithTime(reason, r.clock.Now()) } // rotateWithTime closes the actively written file, and rotates it along with exising @@ -339,7 +327,7 @@ func (r *Rotator) isRotationTriggered(dataLen uint) (rotateReason, time.Time) { for _, t := range r.triggers { reason := t.TriggerRotation(dataLen) if reason != rotateReasonNoRotate { - return reason, time.Now() + return reason, r.clock.Now() } } return rotateReasonNoRotate, time.Time{} @@ -395,52 +383,37 @@ func (r *Rotator) closeFile() error { return errors.Wrap(err, "failed to close active file") } -type countRotator struct { - log Logger - filename string - intervalRotator *intervalRotator - maxBackups uint -} - type dateRotator struct { log Logger + clock clock format string filenamePrefix string currentFilename string - intervalRotator *intervalRotator -} + extension string -func newRotater(log Logger, s SuffixType, filename string, maxBackups uint, interval time.Duration) rotater { - switch s { - case SuffixCount: - if interval > 0 { - return newIntervalRotator(log, interval, filename) - } - return &countRotator{ - log: log, - filename: filename, - maxBackups: maxBackups, - } - case SuffixDate: - return newDateRotater(log, filename) - default: - return &countRotator{ - log: log, - filename: filename, - maxBackups: maxBackups, - } - } + prefixLen int + filenameLen int + extensionLen int + + // logOrderCache is used to cache log file meta information between rotations + logOrderCache map[string]logOrder } -func newDateRotater(log Logger, filename string) rotater { +func newDateRotater(log Logger, filename string, clock clock) rotater { d := &dateRotator{ log: log, + clock: clock, filenamePrefix: filename + "-", - format: "20060102150405", + extension: ".ndjson", + format: DateFormat, + logOrderCache: make(map[string]logOrder), } + d.prefixLen = len(d.filenamePrefix) + d.filenameLen = d.prefixLen + len(DateFormat) + d.extensionLen = len(d.extension) - d.currentFilename = d.filenamePrefix + time.Now().Format(d.format) - files, err := filepath.Glob(d.filenamePrefix + "*") + d.currentFilename = d.filenamePrefix + d.clock.Now().Format(d.format) + d.extension + files, err := filepath.Glob(d.filenamePrefix + "*" + d.extension) if err != nil { return d } @@ -467,7 +440,24 @@ func (d *dateRotator) Rotate(reason rotateReason, rotateTime time.Time) error { d.log.Debugw("Rotating file", "filename", d.currentFilename, "reason", reason) } - d.currentFilename = d.filenamePrefix + rotateTime.Format(d.format) + d.logOrderCache = make(map[string]logOrder, 0) + + newFileNamePrefix := d.filenamePrefix + rotateTime.Format(d.format) + files, err := filepath.Glob(newFileNamePrefix + "*" + d.extension) + if err != nil { + return fmt.Errorf("failed to get possible files: %+v", err) + } + + if len(files) == 0 { + d.currentFilename = newFileNamePrefix + d.extension + return nil + } + + d.SortModTimeLogs(files) + order := d.OrderLog(files[len(files)-1]) + + d.currentFilename = newFileNamePrefix + "-" + strconv.Itoa(order.index+1) + d.extension + return nil } @@ -479,10 +469,18 @@ func (d *dateRotator) RotatedFiles() []string { } } + for i, name := range files { + if name == d.ActiveFile() { + files = append(files[:i], files[i+1:]...) + break + } + } + d.SortModTimeLogs(files) return files } +// SortModTimeLogs puts newest file to the last func (d *dateRotator) SortModTimeLogs(strings []string) { sort.Slice( strings, @@ -492,88 +490,53 @@ func (d *dateRotator) SortModTimeLogs(strings []string) { ) } -func (d *dateRotator) OrderLog(filename string) time.Time { - ts, err := time.Parse(d.filenamePrefix+d.format, filepath.Base(filename)) - if err != nil { - return time.Time{} - } - return ts -} - -func (c *countRotator) ActiveFile() string { - return c.filename +// logOrder stores information required to sort log files +// parsed out from the following format {filename}-{datetime}-{index}.ndjson +type logOrder struct { + index int + datetime time.Time } -func (c *countRotator) RotatedFiles() []string { - files := make([]string, 0) - for i := c.maxBackups + 1; i >= 1; i-- { - name := c.backupName(i) - if _, err := os.Stat(name); os.IsNotExist(err) { - continue - } else if err != nil { - c.log.Debugw("failed to stat rotated file") - return files - } - files = append(files, name) +func (o logOrder) After(other logOrder) bool { + if o.datetime.Equal(other.datetime) { + return other.index > o.index } - - return files + return !o.datetime.After(other.datetime) } -func (c *countRotator) backupName(n uint) string { - if n == 0 { - return c.ActiveFile() +func (d *dateRotator) OrderLog(filename string) logOrder { + if o, ok := d.logOrderCache[filename]; ok { + return o } - return c.ActiveFile() + "." + strconv.Itoa(int(n)) -} -func (c *countRotator) Rotate(reason rotateReason, _ time.Time) error { - for i := c.maxBackups + 1; i > 0; i-- { - old := c.backupName(i - 1) - older := c.backupName(i) + var o logOrder + var err error - if _, err := os.Stat(old); os.IsNotExist(err) { - continue - } else if err != nil { - return errors.Wrap(err, "failed to rotate backups") - } + o.datetime, err = time.Parse(d.format, filename[d.prefixLen:d.filenameLen]) + if err != nil { + return o + } - if err := os.Remove(older); err != nil && !os.IsNotExist(err) { - return errors.Wrap(err, "failed to rotate backups") - } - if err := os.Rename(old, older); err != nil { - return errors.Wrap(err, "failed to rotate backups") - } else if i == 1 { - // Log when rotation of the main file occurs. - if c.log != nil { - c.log.Debugw("Rotating file", "filename", old, "reason", reason) - } + if d.isFilenameWithIndex(filename) { + o.index, err = d.filenameIndex(filename) + if err != nil { + return o } } - return nil -} -func (s *SuffixType) Unpack(v string) error { - i, err := strconv.Atoi(v) - if err == nil { - t := SuffixType(i) - v = t.String() - } + d.logOrderCache[filename] = o - val, ok := suffixes[v] - if !ok { - return fmt.Errorf("invalid suffix type: %+v", v) - } + return o +} - *s = val - return nil +func (d *dateRotator) isFilenameWithIndex(filename string) bool { + return d.filenameLen+d.extensionLen < len(filename) } -func (s *SuffixType) String() string { - for k, v := range suffixes { - if v == *s { - return k - } +func (d *dateRotator) filenameIndex(filename string) (int, error) { + indexStr := filename[d.filenameLen+1 : len(filename)-d.extensionLen] + if len(indexStr) > 0 { + return strconv.Atoi(indexStr) } - return "" + return 0, nil } diff --git a/libbeat/common/file/rotator_test.go b/libbeat/common/file/rotator_test.go index 585d1690e78..6294b042d13 100644 --- a/libbeat/common/file/rotator_test.go +++ b/libbeat/common/file/rotator_test.go @@ -21,8 +21,6 @@ import ( "fmt" "os" "path/filepath" - "regexp" - "sort" "sync" "testing" "time" @@ -39,43 +37,57 @@ func TestFileRotator(t *testing.T) { logp.TestingSetup() dir := t.TempDir() + logname := "sample" + c := &testClock{time.Date(2021, 11, 11, 0, 0, 0, 0, time.Local)} - filename := filepath.Join(dir, "sample.log") + filename := filepath.Join(dir, logname) r, err := file.NewFileRotator(filename, file.MaxBackups(2), file.WithLogger(logp.NewLogger("rotator").With(logp.Namespace("rotator"))), + file.WithClock(c), ) if err != nil { t.Fatal(err) } defer r.Close() + firstFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) + WriteMsg(t, r) - AssertDirContents(t, dir, "sample.log") + AssertDirContents(t, dir, firstFile) + + c.time = time.Date(2021, 11, 12, 0, 0, 0, 0, time.Local) Rotate(t, r) - AssertDirContents(t, dir, "sample.log.1") + AssertDirContents(t, dir, firstFile) WriteMsg(t, r) - AssertDirContents(t, dir, "sample.log", "sample.log.1") + + secondFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) + AssertDirContents(t, dir, firstFile, secondFile) + + c.time = time.Date(2021, 11, 13, 0, 0, 0, 0, time.Local) Rotate(t, r) - AssertDirContents(t, dir, "sample.log.1", "sample.log.2") + AssertDirContents(t, dir, firstFile, secondFile) WriteMsg(t, r) - AssertDirContents(t, dir, "sample.log", "sample.log.1", "sample.log.2") + thirdFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) + AssertDirContents(t, dir, firstFile, secondFile, thirdFile) + c.time = time.Date(2021, 11, 14, 0, 0, 0, 0, time.Local) Rotate(t, r) - AssertDirContents(t, dir, "sample.log.1", "sample.log.2") + AssertDirContents(t, dir, secondFile, thirdFile) + c.time = time.Date(2021, 11, 15, 0, 0, 0, 0, time.Local) Rotate(t, r) - AssertDirContents(t, dir, "sample.log.2", "sample.log.3") + AssertDirContents(t, dir, secondFile, thirdFile) } func TestFileRotatorConcurrently(t *testing.T) { dir := t.TempDir() - filename := filepath.Join(dir, "sample.log") + filename := filepath.Join(dir, "sample") r, err := file.NewFileRotator(filename, file.MaxBackups(2)) if err != nil { t.Fatal(err) @@ -97,29 +109,27 @@ func TestDailyRotation(t *testing.T) { dir := t.TempDir() logname := "daily" - dateFormat := "2006-01-02" - today := time.Now().Format(dateFormat) - yesterday := time.Now().AddDate(0, 0, -1).Format(dateFormat) - twoDaysAgo := time.Now().AddDate(0, 0, -2).Format(dateFormat) + yesterday := time.Now().AddDate(0, 0, -1).Format(file.DateFormat) + twoDaysAgo := time.Now().AddDate(0, 0, -2).Format(file.DateFormat) // seed directory with existing log files files := []string{ - logname + "-" + yesterday + "-1", - logname + "-" + yesterday + "-2", - logname + "-" + yesterday + "-3", - logname + "-" + yesterday + "-4", - logname + "-" + yesterday + "-5", - logname + "-" + yesterday + "-6", - logname + "-" + yesterday + "-7", - logname + "-" + yesterday + "-8", - logname + "-" + yesterday + "-9", - logname + "-" + yesterday + "-10", - logname + "-" + yesterday + "-11", - logname + "-" + yesterday + "-12", - logname + "-" + yesterday + "-13", - logname + "-" + twoDaysAgo + "-1", - logname + "-" + twoDaysAgo + "-2", - logname + "-" + twoDaysAgo + "-3", + logname + "-" + yesterday + "-1.ndjson", + logname + "-" + yesterday + "-2.ndjson", + logname + "-" + yesterday + "-3.ndjson", + logname + "-" + yesterday + "-4.ndjson", + logname + "-" + yesterday + "-5.ndjson", + logname + "-" + yesterday + "-6.ndjson", + logname + "-" + yesterday + "-7.ndjson", + logname + "-" + yesterday + "-8.ndjson", + logname + "-" + yesterday + "-9.ndjson", + logname + "-" + yesterday + "-10.ndjson", + logname + "-" + yesterday + "-11.ndjson", + logname + "-" + yesterday + "-12.ndjson", + logname + "-" + yesterday + "-13.ndjson", + logname + "-" + twoDaysAgo + "-1.ndjson", + logname + "-" + twoDaysAgo + "-2.ndjson", + logname + "-" + twoDaysAgo + "-3.ndjson", } for _, f := range files { @@ -139,25 +149,26 @@ func TestDailyRotation(t *testing.T) { Rotate(t, r) - AssertDirContents(t, dir, logname+"-"+yesterday+"-12", logname+"-"+yesterday+"-13") + AssertDirContents(t, dir, logname+"-"+yesterday+"-12.ndjson", logname+"-"+yesterday+"-13.ndjson") WriteMsg(t, r) - AssertDirContents(t, dir, logname+"-"+yesterday+"-12", logname+"-"+yesterday+"-13", logname) + today := time.Now().Format(file.DateFormat) + AssertDirContents(t, dir, logname+"-"+yesterday+"-12.ndjson", logname+"-"+yesterday+"-13.ndjson", logname+"-"+today+".ndjson") Rotate(t, r) - AssertDirContents(t, dir, logname+"-"+yesterday+"-13", logname+"-"+today+"-1") + AssertDirContents(t, dir, logname+"-"+yesterday+"-13.ndjson", logname+"-"+today+".ndjson") WriteMsg(t, r) - AssertDirContents(t, dir, logname+"-"+yesterday+"-13", logname+"-"+today+"-1", logname) + AssertDirContents(t, dir, logname+"-"+yesterday+"-13.ndjson", logname+"-"+today+".ndjson", logname+"-"+today+"-1.ndjson") for i := 0; i < (int(maxSizeBytes)/len(logMessage))+1; i++ { WriteMsg(t, r) } - AssertDirContents(t, dir, logname+"-"+today+"-1", logname+"-"+today+"-2", logname) + AssertDirContents(t, dir, logname+"-"+today+"-1.ndjson", logname+"-"+today+"-2.ndjson", logname+"-"+today+"-3.ndjson") } // Tests the FileConfig.RotateOnStartup parameter @@ -165,13 +176,15 @@ func TestRotateOnStartup(t *testing.T) { dir := t.TempDir() logname := "rotate_on_open" - filename := filepath.Join(dir, logname) + c := &testClock{time.Date(2021, 11, 11, 0, 0, 0, 0, time.Local)} + firstFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) + filename := filepath.Join(dir, firstFile) // Create an existing log file with this name. CreateFile(t, filename) - AssertDirContents(t, dir, logname) + AssertDirContents(t, dir, firstFile) - r, err := file.NewFileRotator(filename, file.RotateOnStartup(false)) + r, err := file.NewFileRotator(filepath.Join(dir, logname), file.RotateOnStartup(false), file.WithClock(c)) if err != nil { t.Fatal(err) } @@ -179,33 +192,37 @@ func TestRotateOnStartup(t *testing.T) { WriteMsg(t, r) // The line should have been appended to the existing file without rotation. - AssertDirContents(t, dir, logname) + AssertDirContents(t, dir, firstFile) // Close the first rotator early (the deferred close will be a no-op if // we haven't hit an error by now), so it can't interfere with the second one. r.Close() // Create a second rotator with the default setting of rotateOnStartup=true - r, err = file.NewFileRotator(filename) + c = &testClock{time.Date(2021, 11, 12, 0, 0, 0, 0, time.Local)} + r, err = file.NewFileRotator(filepath.Join(dir, logname), file.WithClock(c)) if err != nil { t.Fatal(err) } defer r.Close() // The directory contents shouldn't change until the first Write. - AssertDirContents(t, dir, logname) + AssertDirContents(t, dir, firstFile) + + secondFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) WriteMsg(t, r) - AssertDirContents(t, dir, logname, logname+".1") + AssertDirContents(t, dir, firstFile, secondFile) } -func TestRotateDateSuffix(t *testing.T) { +func TestRotate(t *testing.T) { dir := t.TempDir() logname := "beatname" filename := filepath.Join(dir, logname) - r, err := file.NewFileRotator(filename, file.Suffix(file.SuffixDate), file.MaxBackups(1)) + c := &testClock{time.Date(2021, 11, 11, 0, 0, 0, 0, time.Local)} + r, err := file.NewFileRotator(filename, file.MaxBackups(1), file.WithClock(c)) if err != nil { t.Fatal(err) } @@ -213,24 +230,24 @@ func TestRotateDateSuffix(t *testing.T) { WriteMsg(t, r) - firstExpectedPattern := fmt.Sprintf("%s-%s.*", logname, time.Now().Format("20060102150405")) - AssertDirContentsPattern(t, dir, firstExpectedPattern) + firstFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) + AssertDirContents(t, dir, firstFile) - time.Sleep(2 * time.Second) - secondExpectedPattern := fmt.Sprintf("%s-%s.*", logname, time.Now().Format("20060102150405")) + c.time = time.Date(2021, 11, 13, 0, 0, 0, 0, time.Local) + secondFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) Rotate(t, r) WriteMsg(t, r) - AssertDirContentsPattern(t, dir, firstExpectedPattern, secondExpectedPattern) + AssertDirContents(t, dir, firstFile, secondFile) - time.Sleep(2 * time.Second) - thirdExpectedPattern := fmt.Sprintf("%s-%s.*", logname, time.Now().Format("20060102150405")) + c.time = time.Date(2021, 11, 15, 0, 0, 0, 0, time.Local) + thirdFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) Rotate(t, r) WriteMsg(t, r) - AssertDirContentsPattern(t, dir, secondExpectedPattern, thirdExpectedPattern) + AssertDirContents(t, dir, secondFile, thirdFile) } func CreateFile(t *testing.T, filename string) { @@ -258,36 +275,7 @@ func AssertDirContents(t *testing.T, dir string, files ...string) { t.Fatal(err) } - sort.Strings(files) - sort.Strings(names) - assert.EqualValues(t, files, names) -} - -func AssertDirContentsPattern(t *testing.T, dir string, patterns ...string) { - t.Helper() - - f, err := os.Open(dir) - if err != nil { - t.Fatal(err) - } - - names, err := f.Readdirnames(-1) - if err != nil { - t.Fatal(err) - } - if len(patterns) != len(names) { - t.Fatal("unexpected number of files") - } - - sort.Strings(patterns) - sort.Strings(names) - for i := 0; i < len(patterns); i++ { - matches, err := regexp.MatchString(patterns[i], names[i]) - if err != nil { - t.Fatal(err) - } - assert.True(t, matches, "pattern: %s name: %s", patterns[i], names[i]) - } + assert.ElementsMatch(t, files, names) } func WriteMsg(t *testing.T, r *file.Rotator) { @@ -307,3 +295,11 @@ func Rotate(t *testing.T, r *file.Rotator) { t.Fatal(err) } } + +type testClock struct { + time time.Time +} + +func (t testClock) Now() time.Time { + return t.time +} diff --git a/libbeat/common/file/trigger.go b/libbeat/common/file/trigger.go index d96d748f922..22c128a334b 100644 --- a/libbeat/common/file/trigger.go +++ b/libbeat/common/file/trigger.go @@ -52,14 +52,14 @@ type trigger interface { TriggerRotation(dataLen uint) rotateReason } -func newTriggers(rotateOnStartup bool, interval time.Duration, maxSizeBytes uint) []trigger { +func newTriggers(rotateOnStartup bool, interval time.Duration, maxSizeBytes uint, clock clock) []trigger { triggers := make([]trigger, 0) if rotateOnStartup { triggers = append(triggers, &initTrigger{}) } if interval > 0 { - triggers = append(triggers, newIntervalTrigger(interval)) + triggers = append(triggers, newIntervalTrigger(interval, clock)) } if maxSizeBytes > 0 { triggers = append(triggers, &sizeTrigger{maxSizeBytes: maxSizeBytes, size: 0}) @@ -113,8 +113,8 @@ func (realClock) Now() time.Time { return time.Now() } -func newIntervalTrigger(interval time.Duration) trigger { - t := intervalTrigger{interval: interval, clock: realClock{}} +func newIntervalTrigger(interval time.Duration, clock clock) trigger { + t := intervalTrigger{interval: interval, clock: clock} switch interval { case time.Second: diff --git a/libbeat/docs/loggingconfig.asciidoc b/libbeat/docs/loggingconfig.asciidoc index 4b47c45d5fb..d6232e2cb07 100644 --- a/libbeat/docs/loggingconfig.asciidoc +++ b/libbeat/docs/loggingconfig.asciidoc @@ -246,15 +246,6 @@ Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h are boundary-aligned with minutes, hours, days, weeks, months, and years as reported by the local system clock. All other intervals are calculated from the unix epoch. Defaults to disabled. - -[float] -==== `logging.files.suffix` - -When a log rotation happens it can either rename older files with -an incresing index if `count` is configured. The other option is `date` -that appends the current date and time to the end of the filename. -When the log is rotated a new file is created and older files -remain untouched. endif::serverless[] [float] diff --git a/libbeat/logp/config.go b/libbeat/logp/config.go index c5de43ae51a..1e82b6e7dd8 100644 --- a/libbeat/logp/config.go +++ b/libbeat/logp/config.go @@ -19,8 +19,6 @@ package logp import ( "time" - - "github.com/elastic/beats/v7/libbeat/common/file" ) // Config contains the configuration options for the logger. To create a Config @@ -47,15 +45,14 @@ type Config struct { // FileConfig contains the configuration options for the file output. type FileConfig struct { - Path string `config:"path" yaml:"path"` - Name string `config:"name" yaml:"name"` - Suffix file.SuffixType `config:"suffix" yaml:"suffix"` - MaxSize uint `config:"rotateeverybytes" yaml:"rotateeverybytes" validate:"min=1"` - MaxBackups uint `config:"keepfiles" yaml:"keepfiles" validate:"max=1024"` - Permissions uint32 `config:"permissions"` - Interval time.Duration `config:"interval"` - RotateOnStartup bool `config:"rotateonstartup"` - RedirectStderr bool `config:"redirect_stderr" yaml:"redirect_stderr"` + Path string `config:"path" yaml:"path"` + Name string `config:"name" yaml:"name"` + MaxSize uint `config:"rotateeverybytes" yaml:"rotateeverybytes" validate:"min=1"` + MaxBackups uint `config:"keepfiles" yaml:"keepfiles" validate:"max=1024"` + Permissions uint32 `config:"permissions"` + Interval time.Duration `config:"interval"` + RotateOnStartup bool `config:"rotateonstartup"` + RedirectStderr bool `config:"redirect_stderr" yaml:"redirect_stderr"` } // MetricsConfig contains configuration used by the monitor to output metrics into the logstream. @@ -76,7 +73,6 @@ func DefaultConfig(environment Environment) Config { return Config{ Level: defaultLevel, Files: FileConfig{ - Suffix: file.SuffixCount, MaxSize: 10 * 1024 * 1024, MaxBackups: 7, Permissions: 0600, diff --git a/libbeat/logp/core.go b/libbeat/logp/core.go index c85a5feeaec..552c81e9201 100644 --- a/libbeat/logp/core.go +++ b/libbeat/logp/core.go @@ -241,7 +241,6 @@ func makeFileOutput(cfg Config) (zapcore.Core, error) { file.Interval(cfg.Files.Interval), file.RotateOnStartup(cfg.Files.RotateOnStartup), file.RedirectStderr(cfg.Files.RedirectStderr), - file.Suffix(cfg.Files.Suffix), ) if err != nil { return nil, errors.Wrap(err, "failed to create file rotator") diff --git a/libbeat/logp/core_test.go b/libbeat/logp/core_test.go index 293f57098eb..f8537eb6aa7 100644 --- a/libbeat/logp/core_test.go +++ b/libbeat/logp/core_test.go @@ -153,7 +153,7 @@ func TestLoggingECSFields(t *testing.T) { Level: DebugLevel, development: true, Files: FileConfig{ - Name: "beat1.log", + Name: "beat1", }, } ToObserverOutput()(&cfg) diff --git a/libbeat/outputs/fileout/config.go b/libbeat/outputs/fileout/config.go index 28441c93bb0..e80f0a27be9 100644 --- a/libbeat/outputs/fileout/config.go +++ b/libbeat/outputs/fileout/config.go @@ -25,19 +25,17 @@ import ( ) type config struct { - Path string `config:"path"` - Filename string `config:"filename"` - Suffix file.SuffixType `config:"suffix"` - RotateEveryKb uint `config:"rotate_every_kb" validate:"min=1"` - NumberOfFiles uint `config:"number_of_files"` - Codec codec.Config `config:"codec"` - Permissions uint32 `config:"permissions"` - RotateOnStartup bool `config:"rotate_on_startup"` + Path string `config:"path"` + Filename string `config:"filename"` + RotateEveryKb uint `config:"rotate_every_kb" validate:"min=1"` + NumberOfFiles uint `config:"number_of_files"` + Codec codec.Config `config:"codec"` + Permissions uint32 `config:"permissions"` + RotateOnStartup bool `config:"rotate_on_startup"` } func defaultConfig() config { return config{ - Suffix: file.SuffixCount, NumberOfFiles: 7, RotateEveryKb: 10 * 1024, Permissions: 0600, diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index 48a65adc1c0..b4a10e38396 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -84,7 +84,6 @@ func (out *fileOutput) init(beat beat.Info, c config) error { var err error out.rotator, err = file.NewFileRotator( path, - file.Suffix(c.Suffix), file.MaxSizeBytes(c.RotateEveryKb*1024), file.MaxBackups(c.NumberOfFiles), file.Permissions(os.FileMode(c.Permissions)), diff --git a/libbeat/tests/system/base.py b/libbeat/tests/system/base.py index 7fd9f1dbb49..a768d80e31e 100644 --- a/libbeat/tests/system/base.py +++ b/libbeat/tests/system/base.py @@ -1,9 +1,11 @@ import os +from datetime import datetime from beat.beat import TestCase from elasticsearch import Elasticsearch, NotFoundError class BaseTest(TestCase): + today = datetime.now().strftime("%Y%m%d") @classmethod def setUpClass(self): diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 195741b1d31..e7a2c60640a 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -119,6 +119,7 @@ def __del__(self): class TestCase(unittest.TestCase, ComposeMixin): + today = datetime.now().strftime("%Y%m%d") @classmethod def setUpClass(self): @@ -207,7 +208,7 @@ def start_beat(self, config = self.beat_name + ".yml" if output is None: - output = self.beat_name + ".log" + output = self.beat_name + "-" + self.today + ".ndjson" args = [cmd, "-systemTest"] if os.getenv("TEST_COVERAGE") == "true": @@ -264,7 +265,7 @@ def read_output(self, # Init defaults if output_file is None: - output_file = "output/" + self.beat_name + output_file = "output/" + self.beat_name + "-" + self.today + ".ndjson" jsons = [] with open(os.path.join(self.working_dir, output_file), "r", encoding="utf_8") as f: @@ -288,7 +289,7 @@ def read_output_json(self, output_file=None): # Init defaults if output_file is None: - output_file = "output/" + self.beat_name + output_file = "output/" + self.beat_name + "-" + self.today + ".ndjson" jsons = [] with open(os.path.join(self.working_dir, output_file), "r", encoding="utf_8") as f: @@ -368,7 +369,7 @@ def get_log(self, logfile=None): Returns the log as a string. """ if logfile is None: - logfile = self.beat_name + ".log" + logfile = self.beat_name + "-" + self.today + ".ndjson" with open(os.path.join(self.working_dir, logfile), 'r', encoding="utf_8") as f: data = f.read() @@ -380,7 +381,7 @@ def get_log_lines(self, logfile=None): Returns the log lines as a list of strings """ if logfile is None: - logfile = self.beat_name + ".log" + logfile = self.beat_name + "-" + self.today + ".ndjson" with open(os.path.join(self.working_dir, logfile), 'r', encoding="utf_8") as f: data = f.readlines() @@ -417,8 +418,9 @@ def log_contains_count(self, msg, logfile=None, ignore_case=False): # Init defaults if logfile is None: - logfile = self.beat_name + ".log" + logfile = self.beat_name + "-" + self.today + ".ndjson" + print("logfile", logfile, self.working_dir) try: with open(os.path.join(self.working_dir, logfile), "r", encoding="utf_8") as f: for line in f: @@ -430,7 +432,8 @@ def log_contains_count(self, msg, logfile=None, ignore_case=False): line = line.lower() if line.find(msg) >= 0: counter = counter + 1 - except IOError: + except IOError as e: + print(e) counter = -1 return counter @@ -442,7 +445,7 @@ def log_contains_countmap(self, pattern, capture_group, logfile=None): counts = {} if logfile is None: - logfile = self.beat_name + ".log" + logfile = self.beat_name + "-" + self.today + ".ndjson" try: with open(os.path.join(self.working_dir, logfile), "r", encoding="utf_8") as f: @@ -462,7 +465,7 @@ def log_contains_countmap(self, pattern, capture_group, logfile=None): def output_lines(self, output_file=None): """ Count number of lines in a file.""" if output_file is None: - output_file = "output/" + self.beat_name + output_file = "output/" + self.beat_name + "-" + self.today + ".ndjson" try: with open(os.path.join(self.working_dir, output_file), "r", encoding="utf_8") as f: @@ -477,7 +480,7 @@ def output_has(self, lines, output_file=None): # Init defaults if output_file is None: - output_file = "output/" + self.beat_name + output_file = "output/" + self.beat_name + "-" + self.today + ".ndjson" try: with open(os.path.join(self.working_dir, output_file, ), "r", encoding="utf_8") as f: @@ -492,7 +495,7 @@ def output_is_empty(self, output_file=None): # Init defaults if output_file is None: - output_file = "output/" + self.beat_name + output_file = "output/" + self.beat_name + "-" + self.today + ".ndjson" try: with open(os.path.join(self.working_dir, output_file, ), "r", encoding="utf_8") as f: @@ -656,7 +659,7 @@ def output_count(self, pred, output_file=None): # Init defaults if output_file is None: - output_file = "output/" + self.beat_name + output_file = "output/" + self.beat_name + "-" + self.today + ".ndjson" try: with open(os.path.join(self.working_dir, output_file), "r", encoding="utf_8") as f: diff --git a/libbeat/tests/system/test_base.py b/libbeat/tests/system/test_base.py index 97615e7ec00..fbb8b324f16 100644 --- a/libbeat/tests/system/test_base.py +++ b/libbeat/tests/system/test_base.py @@ -165,8 +165,7 @@ def run(): # remove log, restart beat and check meta file did not change # and same UUID is used in log output. - - os.remove(os.path.join(self.working_dir, "mockbeat.log")) + os.remove(os.path.join(self.working_dir, "mockbeat-" + self.today + ".ndjson")) meta1 = run() assert self.log_contains("Beat ID: {}".format(meta1["uuid"])) diff --git a/libbeat/tests/system/test_logging.py b/libbeat/tests/system/test_logging.py index 4295bfd1776..4380e25b55c 100644 --- a/libbeat/tests/system/test_logging.py +++ b/libbeat/tests/system/test_logging.py @@ -39,11 +39,11 @@ def test_file_default(self): """ self.run_beat_with_args("Mockbeat is alive!", logging_args=[]) - self.assert_contains_ecs_log(logfile="logs/mockbeat") + self.assert_contains_ecs_log(logfile="logs/mockbeat-"+self.today+".ndjson") def test_file_ecs(self): """ logs to file with ECS format """ self.run_beat_with_args("Mockbeat is alive!") - self.assert_contains_ecs_log(logfile="logs/mockbeat") + self.assert_contains_ecs_log(logfile="logs/mockbeat-"+self.today+".ndjson") diff --git a/libbeat/tests/system/test_umask.py b/libbeat/tests/system/test_umask.py index e78214e2d9c..d6a7b7f391c 100644 --- a/libbeat/tests/system/test_umask.py +++ b/libbeat/tests/system/test_umask.py @@ -30,7 +30,7 @@ def test_output_file_perms(self): """ Test that output file permissions respect default umask """ - output_file_path = os.path.join(self.working_dir, "output", "mockbeat") + output_file_path = os.path.join(self.working_dir, "output", "mockbeat-" + self.today + ".ndjson") perms = stat.S_IMODE(os.lstat(output_file_path).st_mode) self.assertEqual(perms, self.output_file_permissions & ~TestUmask.DEFAULT_UMASK) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index d78e42dcf8c..178a5cb2411 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -2266,11 +2266,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. metricbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 3377abc029a..a34efacdc33 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1938,11 +1938,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. packetbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Packetbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/packetbeat/tests/system/packetbeat.py b/packetbeat/tests/system/packetbeat.py index 32a9366ee33..c4727d13183 100644 --- a/packetbeat/tests/system/packetbeat.py +++ b/packetbeat/tests/system/packetbeat.py @@ -33,7 +33,7 @@ def setUpClass(self): def run_packetbeat(self, pcap, cmd=None, config="packetbeat.yml", - output="packetbeat.log", + output=None, extra_args=[], debug_selectors=[], exit_code=0, @@ -44,6 +44,9 @@ def run_packetbeat(self, pcap, the caller. """ + if output is None: + output = "packetbeat-" + self.today + ".ndjson" + if cmd is None: cmd = self.beat_path + "/packetbeat.test" @@ -87,7 +90,7 @@ def run_packetbeat(self, pcap, def start_packetbeat(self, cmd=None, config="packetbeat.yml", - output="packetbeat.log", + output=None, extra_args=[], debug_selectors=[]): """ @@ -95,6 +98,9 @@ def start_packetbeat(self, caller is responsible for stopping / waiting for the Proc instance. """ + if output is None: + output = "packetbeat-" + self.today + ".ndjson" + if cmd is None: cmd = self.beat_path + "/packetbeat.test" @@ -119,9 +125,14 @@ def start_packetbeat(self, return proc def read_output(self, - output_file="output/packetbeat", + output_file=None, types=None, required_fields=None): + + if output_file is None: + output_file = "output/packetbeat-"+self.today+".ndjson" + print(output_file) + jsons = [] with open(os.path.join(self.working_dir, output_file), "r", encoding='utf_8') as f: for line in f: diff --git a/packetbeat/tests/system/test_0002_thrift_basics.py b/packetbeat/tests/system/test_0002_thrift_basics.py index 6e0154a437c..a1a66742bdc 100644 --- a/packetbeat/tests/system/test_0002_thrift_basics.py +++ b/packetbeat/tests/system/test_0002_thrift_basics.py @@ -208,7 +208,7 @@ def test_thrift_send_request_response(self): self.run_packetbeat(pcap="thrift_integration.pcap", debug_selectors=["thrift"]) - objs = self.read_output() + objs = self.read_output(output_file="output/packetbeat-" + self.today + "-1.ndjson") assert all(["request" not in o for o in objs]) assert all(["response" not in o for o in objs]) diff --git a/packetbeat/tests/system/test_0006_wsgi.py b/packetbeat/tests/system/test_0006_wsgi.py index 6f905942449..02e52b885a3 100644 --- a/packetbeat/tests/system/test_0006_wsgi.py +++ b/packetbeat/tests/system/test_0006_wsgi.py @@ -100,7 +100,7 @@ def test_send_headers_options(self): ) self.run_packetbeat(pcap="wsgi_loopback.pcap") - objs = self.read_output() + objs = self.read_output(output_file="output/packetbeat-" + self.today + "-1.ndjson") assert len(objs) == 1 o = objs[0] @@ -119,7 +119,7 @@ def test_send_headers_options(self): ) self.run_packetbeat(pcap="wsgi_loopback.pcap") - objs = self.read_output() + objs = self.read_output(output_file="output/packetbeat-" + self.today + "-2.ndjson") assert len(objs) == 1 o = objs[0] diff --git a/winlogbeat/tests/system/test_config.py b/winlogbeat/tests/system/test_config.py index 306f6cf38ef..aefab4344ba 100644 --- a/winlogbeat/tests/system/test_config.py +++ b/winlogbeat/tests/system/test_config.py @@ -82,7 +82,7 @@ def run_config_tst(self, pcap=None, exit_code=0): args.extend(["test", "config"]) - output = "winlogbeat.log" + output = "winlogbeat-" + self.today + ".ndjson" with open(os.path.join(self.working_dir, output), "wb") as outfile: proc = subprocess.Popen(args, diff --git a/winlogbeat/tests/system/test_wineventlog.py b/winlogbeat/tests/system/test_wineventlog.py index 347b8585d05..9bbf5a7ded4 100644 --- a/winlogbeat/tests/system/test_wineventlog.py +++ b/winlogbeat/tests/system/test_wineventlog.py @@ -52,7 +52,7 @@ def test_resume_reading_events(self): # remove the output file, otherwise there is a race condition # in read_events() below where it reads the results of the previous # execution - os.unlink(os.path.join(self.working_dir, "output", self.beat_name)) + os.unlink(os.path.join(self.working_dir, "output", self.beat_name + "-" + self.today + ".ndjson")) msg = "Second event" self.write_event_log(msg) diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 77886958368..1705648185a 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -1366,11 +1366,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. winlogbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index bab59c339c1..c5f30eecbe0 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -1499,11 +1499,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. auditbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index dedec28762a..19cd40cda91 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -234,10 +234,8 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa }, }, "paths": []string{ - filepath.Join(paths.Home(), "logs", "elastic-agent-json.log"), - filepath.Join(paths.Home(), "logs", "elastic-agent-json.log*"), - filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log"), - filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log*"), + filepath.Join(paths.Home(), "logs", "elastic-agent-*.ndjson"), + filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-*.ndjson"), }, "index": fmt.Sprintf("logs-elastic_agent-%s", monitoringNamespace), "processors": []map[string]interface{}{ diff --git a/x-pack/elastic-agent/pkg/core/logger/logger.go b/x-pack/elastic-agent/pkg/core/logger/logger.go index 77cc4260acc..a63bc94f9de 100644 --- a/x-pack/elastic-agent/pkg/core/logger/logger.go +++ b/x-pack/elastic-agent/pkg/core/logger/logger.go @@ -103,7 +103,6 @@ func DefaultLoggingConfig() *Config { cfg.ToFiles = true cfg.Files.Path = paths.Logs() cfg.Files.Name = agentName - cfg.Files.Suffix = file.SuffixDate return &cfg } @@ -115,7 +114,7 @@ func makeInternalFileOutput(cfg *Config) (zapcore.Core, error) { // defaultCfg is used to set the defaults for the file rotation of the internal logging // these settings cannot be changed by a user configuration defaultCfg := logp.DefaultConfig(logp.DefaultEnvironment) - filename := filepath.Join(paths.Home(), "logs", fmt.Sprintf("%s-json.log", cfg.Beat)) + filename := filepath.Join(paths.Home(), "logs", cfg.Beat) rotator, err := file.NewFileRotator(filename, file.MaxSizeBytes(defaultCfg.Files.MaxSize), @@ -124,7 +123,6 @@ func makeInternalFileOutput(cfg *Config) (zapcore.Core, error) { file.Interval(defaultCfg.Files.Interval), file.RotateOnStartup(defaultCfg.Files.RotateOnStartup), file.RedirectStderr(defaultCfg.Files.RedirectStderr), - file.Suffix(cfg.Files.Suffix), ) if err != nil { return nil, errors.New("failed to create internal file rotator") diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go index f96f5ecf916..e00860ec1f1 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go @@ -131,7 +131,7 @@ func (b *Monitor) EnrichArgs(spec program.Spec, pipelineID string, args []string if isSidecar { logFile += "_monitor" } - logFile = fmt.Sprintf("%s-json.log", logFile) + logFile = fmt.Sprintf("%s", logFile) appendix = append(appendix, "-E", "logging.files.path="+loggingPath, "-E", "logging.files.name="+logFile, diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go index 240ce5adbb2..d56aee0ef2b 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go @@ -16,9 +16,9 @@ import ( const ( // args: data path, pipeline name, application name - logFileFormat = "%s/logs/%s/%s-json.log" + logFileFormat = "%s/logs/%s/%s" // args: data path, install path, pipeline name, application name - logFileFormatWin = "%s\\logs\\%s\\%s-json.log" + logFileFormatWin = "%s\\logs\\%s\\%s" // args: pipeline name, application name mbEndpointFileFormatWin = `npipe:///%s-%s` diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 428230017f7..08c3cf4bb79 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -4508,11 +4508,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. filebeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 4b6dab7943f..397214f8cd5 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -1237,11 +1237,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. functionbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Functionbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 1eb37188303..6bac78d08c2 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -1589,11 +1589,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. heartbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Heartbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index bcece50b832..575581600bd 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -2787,11 +2787,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. metricbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index 13dfcc2089e..c22416ddb19 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -956,11 +956,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. osquerybeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Osquerybeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 3377abc029a..a34efacdc33 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -1938,11 +1938,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. packetbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Packetbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 09afd2e6208..982b46591ec 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -1409,11 +1409,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. winlogbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The