From fa6207d1b744eac1dbf45aa7b8f165ec6275abbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 8 Nov 2021 13:46:13 +0100 Subject: [PATCH 01/19] Initialize changes --- libbeat/common/file/rotator.go | 132 +-------------------------------- 1 file changed, 1 insertion(+), 131 deletions(-) diff --git a/libbeat/common/file/rotator.go b/libbeat/common/file/rotator.go index 633f94232ef5..5573c4e05730 100644 --- a/libbeat/common/file/rotator.go +++ b/libbeat/common/file/rotator.go @@ -22,29 +22,18 @@ import ( "os" "path/filepath" "sort" - "strconv" "sync" "time" "github.com/pkg/errors" ) -type SuffixType uint32 - const ( // MaxBackupsLimit is the upper bound on the number of backup files. Any values // greater will result in an error. MaxBackupsLimit = 1024 - - SuffixCount SuffixType = iota + 1 - SuffixDate ) -var suffixes = map[string]SuffixType{ - "count": SuffixCount, - "date": SuffixDate, -} - // rotater is the interface responsible for rotating and finding files. type rotater interface { // ActiveFile returns the path to the file that is actively written. @@ -69,7 +58,6 @@ type Rotator struct { interval time.Duration permissions os.FileMode log Logger // Optional Logger (may be nil). - suffix SuffixType rotateOnStartup bool redirectStderr bool @@ -85,14 +73,6 @@ type Logger interface { // RotatorOption is a configuration option for Rotator. type RotatorOption func(r *Rotator) -// Interval sets the time interval for log rotation in addition to log -// rotation by size. The default is 0 for disabled. -func Suffix(s SuffixType) RotatorOption { - return func(r *Rotator) { - r.suffix = s - } -} - // MaxSizeBytes configures the maximum number of bytes that a file should // contain before being rotated. The default is 10 MiB. func MaxSizeBytes(n uint) RotatorOption { @@ -158,7 +138,6 @@ func NewFileRotator(filename string, options ...RotatorOption) (*Rotator, error) permissions: 0600, interval: 0, rotateOnStartup: true, - suffix: SuffixCount, } for _, opt := range options { @@ -179,7 +158,7 @@ func NewFileRotator(filename string, options ...RotatorOption) (*Rotator, error) return nil, errors.New("the minimum time interval for log rotation is 1 second") } - r.rot = newRotater(r.log, r.suffix, filename, r.maxBackups, r.interval) + r.rot = newDateRotater(r.log, filename) shouldRotateOnStart := r.rotateOnStartup if _, err := os.Stat(r.rot.ActiveFile()); os.IsNotExist(err) { @@ -194,7 +173,6 @@ func NewFileRotator(filename string, options ...RotatorOption) (*Rotator, error) "max_size_bytes", r.maxSizeBytes, "max_backups", r.maxBackups, "permissions", r.permissions, - "suffix", r.suffix, ) } @@ -395,41 +373,11 @@ func (r *Rotator) closeFile() error { return errors.Wrap(err, "failed to close active file") } -type countRotator struct { - log Logger - filename string - intervalRotator *intervalRotator - maxBackups uint -} - type dateRotator struct { log Logger format string filenamePrefix string currentFilename string - intervalRotator *intervalRotator -} - -func newRotater(log Logger, s SuffixType, filename string, maxBackups uint, interval time.Duration) rotater { - switch s { - case SuffixCount: - if interval > 0 { - return newIntervalRotator(log, interval, filename) - } - return &countRotator{ - log: log, - filename: filename, - maxBackups: maxBackups, - } - case SuffixDate: - return newDateRotater(log, filename) - default: - return &countRotator{ - log: log, - filename: filename, - maxBackups: maxBackups, - } - } } func newDateRotater(log Logger, filename string) rotater { @@ -499,81 +447,3 @@ func (d *dateRotator) OrderLog(filename string) time.Time { } return ts } - -func (c *countRotator) ActiveFile() string { - return c.filename -} - -func (c *countRotator) RotatedFiles() []string { - files := make([]string, 0) - for i := c.maxBackups + 1; i >= 1; i-- { - name := c.backupName(i) - if _, err := os.Stat(name); os.IsNotExist(err) { - continue - } else if err != nil { - c.log.Debugw("failed to stat rotated file") - return files - } - files = append(files, name) - } - - return files -} - -func (c *countRotator) backupName(n uint) string { - if n == 0 { - return c.ActiveFile() - } - return c.ActiveFile() + "." + strconv.Itoa(int(n)) -} - -func (c *countRotator) Rotate(reason rotateReason, _ time.Time) error { - for i := c.maxBackups + 1; i > 0; i-- { - old := c.backupName(i - 1) - older := c.backupName(i) - - if _, err := os.Stat(old); os.IsNotExist(err) { - continue - } else if err != nil { - return errors.Wrap(err, "failed to rotate backups") - } - - if err := os.Remove(older); err != nil && !os.IsNotExist(err) { - return errors.Wrap(err, "failed to rotate backups") - } - if err := os.Rename(old, older); err != nil { - return errors.Wrap(err, "failed to rotate backups") - } else if i == 1 { - // Log when rotation of the main file occurs. - if c.log != nil { - c.log.Debugw("Rotating file", "filename", old, "reason", reason) - } - } - } - return nil -} - -func (s *SuffixType) Unpack(v string) error { - i, err := strconv.Atoi(v) - if err == nil { - t := SuffixType(i) - v = t.String() - } - - val, ok := suffixes[v] - if !ok { - return fmt.Errorf("invalid suffix type: %+v", v) - } - - *s = val - return nil -} - -func (s *SuffixType) String() string { - for k, v := range suffixes { - if v == *s { - return k - } - } - return "" -} From 3d412ce8f09f97e806e6ed2f2c6006bcadb5b63c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 11 Nov 2021 09:20:32 +0100 Subject: [PATCH 02/19] init more --- libbeat/common/file/interval_rotator.go | 192 ------------------- libbeat/common/file/interval_rotator_test.go | 154 --------------- libbeat/common/file/rotator.go | 24 ++- libbeat/common/file/rotator_test.go | 52 +++-- libbeat/logp/config.go | 20 +- libbeat/logp/core.go | 1 - libbeat/outputs/fileout/config.go | 16 +- libbeat/outputs/fileout/file.go | 1 - 8 files changed, 71 insertions(+), 389 deletions(-) delete mode 100644 libbeat/common/file/interval_rotator.go delete mode 100644 libbeat/common/file/interval_rotator_test.go diff --git a/libbeat/common/file/interval_rotator.go b/libbeat/common/file/interval_rotator.go deleted file mode 100644 index e5105beb0a77..000000000000 --- a/libbeat/common/file/interval_rotator.go +++ /dev/null @@ -1,192 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package file - -import ( - "fmt" - "os" - "path/filepath" - "sort" - "strconv" - "time" - - "github.com/pkg/errors" -) - -type intervalRotator struct { - log Logger - interval time.Duration - lastRotate time.Time - filename string - fileFormat string - clock clock - weekly bool - arbitrary bool -} - -func newIntervalRotator(log Logger, interval time.Duration, filename string) rotater { - ir := &intervalRotator{ - filename: filename, - log: log, - interval: (interval / time.Second) * time.Second, // drop fractional seconds - clock: realClock{}, - } - ir.initialize() - return ir -} - -func (r *intervalRotator) initialize() { - switch r.interval { - case time.Second: - r.fileFormat = "2006-01-02-15-04-05" - case time.Minute: - r.fileFormat = "2006-01-02-15-04" - case time.Hour: - r.fileFormat = "2006-01-02-15" - case 24 * time.Hour: // calendar day - r.fileFormat = "2006-01-02" - case 7 * 24 * time.Hour: // calendar week - r.fileFormat = "" - r.weekly = true - case 30 * 24 * time.Hour: // calendar month - r.fileFormat = "2006-01" - case 365 * 24 * time.Hour: // calendar year - r.fileFormat = "2006" - default: - r.arbitrary = true - r.fileFormat = "2006-01-02-15-04-05" - } - - fi, err := os.Stat(r.filename) - if err != nil { - if r.log != nil { - r.log.Debugw("Not attempting to find last rotated time, configured logs dir cannot be opened: %v", err) - } - return - } - r.lastRotate = fi.ModTime() -} - -func (r *intervalRotator) ActiveFile() string { - return r.filename -} - -func (r *intervalRotator) LogPrefix(filename string, modTime time.Time) string { - var t time.Time - if r.lastRotate.IsZero() { - t = modTime - } else { - t = r.lastRotate - } - - if r.weekly { - y, w := t.ISOWeek() - return fmt.Sprintf("%s-%04d-%02d-", filename, y, w) - } - if r.arbitrary { - intervalNumber := t.Unix() / (int64(r.interval) / int64(time.Second)) - intervalStart := time.Unix(0, intervalNumber*int64(r.interval)) - return fmt.Sprintf("%s-%s-", filename, intervalStart.Format(r.fileFormat)) - } - return fmt.Sprintf("%s-%s-", filename, t.Format(r.fileFormat)) -} - -func (r *intervalRotator) RotatedFiles() []string { - files, err := filepath.Glob(r.filename + "*") - if err != nil { - if r.log != nil { - r.log.Debugw("failed to list existing logs: %+v", err) - } - } - r.SortIntervalLogs(files) - return files -} - -func (r *intervalRotator) Rotate(reason rotateReason, t time.Time) error { - fi, err := os.Stat(r.ActiveFile()) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return errors.Wrap(err, "failed to rotate backups") - } - - logPrefix := r.LogPrefix(r.ActiveFile(), fi.ModTime()) - files, err := filepath.Glob(logPrefix + "*") - if err != nil { - return errors.Wrap(err, "failed to list logs during rotation") - } - - var targetFilename string - if len(files) == 0 { - targetFilename = logPrefix + "1" - } else { - r.SortIntervalLogs(files) - lastLogIndex, _, err := IntervalLogIndex(files[len(files)-1]) - if err != nil { - return errors.Wrap(err, "failed to locate last log index during rotation") - } - targetFilename = logPrefix + strconv.Itoa(int(lastLogIndex)+1) - } - - if err := os.Rename(r.ActiveFile(), targetFilename); err != nil { - return errors.Wrap(err, "failed to rotate backups") - } - - if r.log != nil { - r.log.Debugw("Rotating file", "filename", r.ActiveFile(), "reason", reason) - } - - r.lastRotate = t - return nil -} - -func (r *intervalRotator) SortIntervalLogs(strings []string) { - sort.Slice( - strings, - func(i, j int) bool { - return OrderIntervalLogs(strings[i]) < OrderIntervalLogs(strings[j]) - }, - ) -} - -// OrderIntervalLogs, when given a log filename in the form [prefix]-[formattedDate]-n -// returns the filename after zero-padding the trailing n so that foo-[date]-2 sorts -// before foo-[date]-10. -func OrderIntervalLogs(filename string) string { - index, i, err := IntervalLogIndex(filename) - if err == nil { - return filename[:i] + fmt.Sprintf("%020d", index) - } - - return "" -} - -// IntervalLogIndex returns n as int given a log filename in the form [prefix]-[formattedDate]-n -func IntervalLogIndex(filename string) (uint64, int, error) { - i := len(filename) - 1 - for ; i >= 0; i-- { - if '0' > filename[i] || filename[i] > '9' { - break - } - } - i++ - - s64 := filename[i:] - u64, err := strconv.ParseUint(s64, 10, 64) - return u64, i, err -} diff --git a/libbeat/common/file/interval_rotator_test.go b/libbeat/common/file/interval_rotator_test.go deleted file mode 100644 index 729d8c713b7a..000000000000 --- a/libbeat/common/file/interval_rotator_test.go +++ /dev/null @@ -1,154 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package file - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestSecondRotator(t *testing.T) { - a := newMockIntervalRotator(time.Second) - - clock := &testClock{time.Date(2018, 12, 31, 0, 0, 1, 100, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-00-00-01-", a.LogPrefix("foo", time.Now())) -} - -func TestMinuteRotator(t *testing.T) { - a := newMockIntervalRotator(time.Minute) - - clock := &testClock{time.Date(2018, 12, 31, 0, 1, 1, 0, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-00-01-", a.LogPrefix("foo", time.Now())) -} - -func TestHourlyRotator(t *testing.T) { - a := newMockIntervalRotator(time.Hour) - - clock := &testClock{time.Date(2018, 12, 31, 1, 0, 1, 0, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-01-", a.LogPrefix("foo", time.Now())) -} - -func TestDailyRotator(t *testing.T) { - a := newMockIntervalRotator(24 * time.Hour) - - clock := &testClock{time.Date(2018, 12, 31, 0, 0, 0, 0, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-", a.LogPrefix("foo", time.Now())) -} - -func TestWeeklyRotator(t *testing.T) { - a := newMockIntervalRotator(7 * 24 * time.Hour) - - // Monday, 2018-Dec-31 - clock := &testClock{time.Date(2018, 12, 31, 0, 0, 0, 0, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2019-01-", a.LogPrefix("foo", time.Now())) - - // Monday, 2019-Jan-7 - clock.time = clock.time.Add(7 * 24 * time.Hour) - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2019-02-", a.LogPrefix("foo", time.Now())) -} - -func TestMonthlyRotator(t *testing.T) { - a := newMockIntervalRotator(30 * 24 * time.Hour) - - clock := &testClock{time.Date(2018, 12, 1, 0, 0, 0, 0, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(30 * 24 * time.Hour) - assert.Equal(t, "foo-2018-12-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(24 * time.Hour) - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2019-01-", a.LogPrefix("foo", time.Now())) -} - -func TestYearlyRotator(t *testing.T) { - a := newMockIntervalRotator(365 * 24 * time.Hour) - - clock := &testClock{time.Date(2018, 12, 31, 0, 0, 0, 0, time.Local)} - a.clock = clock - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(23 * time.Hour) - assert.Equal(t, "foo-2018-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Hour) - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2019-", a.LogPrefix("foo", time.Now())) -} - -func TestArbitraryIntervalRotator(t *testing.T) { - a := newMockIntervalRotator(3 * time.Second) - - // Monday, 2018-Dec-31 - clock := &testClock{time.Date(2018, 12, 31, 0, 0, 1, 0, time.Local)} - a.clock = clock - assert.Equal(t, "foo-2018-12-30-00-00-00-", a.LogPrefix("foo", time.Date(2018, 12, 30, 0, 0, 0, 0, time.Local))) - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-00-00-00-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - assert.Equal(t, "foo-2018-12-31-00-00-00-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-00-00-03-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - assert.Equal(t, "foo-2018-12-31-00-00-03-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - assert.Equal(t, "foo-2018-12-31-00-00-03-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - a.lastRotate = a.clock.Now() - assert.Equal(t, "foo-2018-12-31-00-00-06-", a.LogPrefix("foo", time.Now())) -} - -func TestIntervalIsTruncatedToSeconds(t *testing.T) { - a := newMockIntervalRotator(2345 * time.Millisecond) - assert.Equal(t, 2*time.Second, a.interval) -} - -type testClock struct { - time time.Time -} - -func (t testClock) Now() time.Time { - return t.time -} - -func newMockIntervalRotator(interval time.Duration) *intervalRotator { - r := newIntervalRotator(nil, interval, "foo").(*intervalRotator) - return r -} diff --git a/libbeat/common/file/rotator.go b/libbeat/common/file/rotator.go index 5573c4e05730..2a1be3ced12a 100644 --- a/libbeat/common/file/rotator.go +++ b/libbeat/common/file/rotator.go @@ -60,6 +60,7 @@ type Rotator struct { log Logger // Optional Logger (may be nil). rotateOnStartup bool redirectStderr bool + clock clock file *os.File mutex sync.Mutex @@ -130,6 +131,12 @@ func RedirectStderr(redirect bool) RotatorOption { } } +func WithClock(clock clock) RotatorOption { + return func(r *Rotator) { + r.clock = clock + } +} + // NewFileRotator returns a new Rotator. func NewFileRotator(filename string, options ...RotatorOption) (*Rotator, error) { r := &Rotator{ @@ -138,6 +145,7 @@ func NewFileRotator(filename string, options ...RotatorOption) (*Rotator, error) permissions: 0600, interval: 0, rotateOnStartup: true, + clock: &realClock{}, } for _, opt := range options { @@ -158,7 +166,7 @@ func NewFileRotator(filename string, options ...RotatorOption) (*Rotator, error) return nil, errors.New("the minimum time interval for log rotation is 1 second") } - r.rot = newDateRotater(r.log, filename) + r.rot = newDateRotater(r.log, filename, r.clock) shouldRotateOnStart := r.rotateOnStartup if _, err := os.Stat(r.rot.ActiveFile()); os.IsNotExist(err) { @@ -270,7 +278,7 @@ func (r *Rotator) openFile() error { } func (r *Rotator) rotate(reason rotateReason) error { - return r.rotateWithTime(reason, time.Now()) + return r.rotateWithTime(reason, r.clock.Now()) } // rotateWithTime closes the actively written file, and rotates it along with exising @@ -375,20 +383,24 @@ func (r *Rotator) closeFile() error { type dateRotator struct { log Logger + clock clock format string filenamePrefix string currentFilename string + extension string } -func newDateRotater(log Logger, filename string) rotater { +func newDateRotater(log Logger, filename string, clock clock) rotater { d := &dateRotator{ log: log, + clock: clock, filenamePrefix: filename + "-", + extension: ".ndjson", format: "20060102150405", } - d.currentFilename = d.filenamePrefix + time.Now().Format(d.format) - files, err := filepath.Glob(d.filenamePrefix + "*") + d.currentFilename = d.filenamePrefix + d.clock.Now().Format(d.format) + d.extension + files, err := filepath.Glob(d.filenamePrefix + "*" + d.extension) if err != nil { return d } @@ -415,7 +427,7 @@ func (d *dateRotator) Rotate(reason rotateReason, rotateTime time.Time) error { d.log.Debugw("Rotating file", "filename", d.currentFilename, "reason", reason) } - d.currentFilename = d.filenamePrefix + rotateTime.Format(d.format) + d.currentFilename = d.filenamePrefix + rotateTime.Format(d.format) + d.extension return nil } diff --git a/libbeat/common/file/rotator_test.go b/libbeat/common/file/rotator_test.go index 585d1690e788..6af7ef625868 100644 --- a/libbeat/common/file/rotator_test.go +++ b/libbeat/common/file/rotator_test.go @@ -35,41 +35,59 @@ import ( const logMessage = "Test file rotator.\n" -func TestFileRotator(t *testing.T) { +func TestFileRotatorB(t *testing.T) { logp.TestingSetup() dir := t.TempDir() + logname := "sample" + testClock := &testClock{time.Date(2021, 11, 11, 11, 12, 0, 0, time.Local)} - filename := filepath.Join(dir, "sample.log") + filename := filepath.Join(dir, logname) r, err := file.NewFileRotator(filename, file.MaxBackups(2), file.WithLogger(logp.NewLogger("rotator").With(logp.Namespace("rotator"))), + file.WithClock(testClock), ) if err != nil { t.Fatal(err) } defer r.Close() + expectedPatterns := []string{ + fmt.Sprintf("%s-%s.ndjson", logname, testClock.Now().Format("20060102150405")), + } + WriteMsg(t, r) - AssertDirContents(t, dir, "sample.log") + AssertDirContents(t, dir, expectedPatterns...) + + testClock.time = time.Date(2021, 11, 11, 11, 13, 0, 0, time.Local) Rotate(t, r) - AssertDirContents(t, dir, "sample.log.1") + AssertDirContents(t, dir, expectedPatterns...) WriteMsg(t, r) - AssertDirContents(t, dir, "sample.log", "sample.log.1") + + expectedPatterns = append(expectedPatterns, fmt.Sprintf("%s-%s.ndjson", logname, testClock.Now().Format("20060102150405"))) + AssertDirContents(t, dir, expectedPatterns...) + + testClock.time = time.Date(2021, 11, 11, 11, 14, 0, 0, time.Local) Rotate(t, r) - AssertDirContents(t, dir, "sample.log.1", "sample.log.2") + AssertDirContents(t, dir, expectedPatterns...) WriteMsg(t, r) - AssertDirContents(t, dir, "sample.log", "sample.log.1", "sample.log.2") + expectedPatterns = append(expectedPatterns, fmt.Sprintf("%s-%s.ndjson", logname, testClock.Now().Format("20060102150405"))) + AssertDirContents(t, dir, expectedPatterns...) + testClock.time = time.Date(2021, 11, 11, 11, 15, 0, 0, time.Local) Rotate(t, r) - AssertDirContents(t, dir, "sample.log.1", "sample.log.2") + expectedPatterns = expectedPatterns[1:] + AssertDirContents(t, dir, expectedPatterns...) + testClock.time = time.Date(2021, 11, 11, 11, 16, 0, 0, time.Local) Rotate(t, r) - AssertDirContents(t, dir, "sample.log.2", "sample.log.3") + expectedPatterns = expectedPatterns[1:] + AssertDirContents(t, dir, expectedPatterns...) } func TestFileRotatorConcurrently(t *testing.T) { @@ -199,13 +217,13 @@ func TestRotateOnStartup(t *testing.T) { AssertDirContents(t, dir, logname, logname+".1") } -func TestRotateDateSuffix(t *testing.T) { +func TestRotateDat(t *testing.T) { dir := t.TempDir() logname := "beatname" filename := filepath.Join(dir, logname) - r, err := file.NewFileRotator(filename, file.Suffix(file.SuffixDate), file.MaxBackups(1)) + r, err := file.NewFileRotator(filename, file.MaxBackups(1)) if err != nil { t.Fatal(err) } @@ -258,9 +276,7 @@ func AssertDirContents(t *testing.T, dir string, files ...string) { t.Fatal(err) } - sort.Strings(files) - sort.Strings(names) - assert.EqualValues(t, files, names) + assert.ElementsMatch(t, files, names) } func AssertDirContentsPattern(t *testing.T, dir string, patterns ...string) { @@ -307,3 +323,11 @@ func Rotate(t *testing.T, r *file.Rotator) { t.Fatal(err) } } + +type testClock struct { + time time.Time +} + +func (t testClock) Now() time.Time { + return t.time +} diff --git a/libbeat/logp/config.go b/libbeat/logp/config.go index c5de43ae51a9..1e82b6e7dd88 100644 --- a/libbeat/logp/config.go +++ b/libbeat/logp/config.go @@ -19,8 +19,6 @@ package logp import ( "time" - - "github.com/elastic/beats/v7/libbeat/common/file" ) // Config contains the configuration options for the logger. To create a Config @@ -47,15 +45,14 @@ type Config struct { // FileConfig contains the configuration options for the file output. type FileConfig struct { - Path string `config:"path" yaml:"path"` - Name string `config:"name" yaml:"name"` - Suffix file.SuffixType `config:"suffix" yaml:"suffix"` - MaxSize uint `config:"rotateeverybytes" yaml:"rotateeverybytes" validate:"min=1"` - MaxBackups uint `config:"keepfiles" yaml:"keepfiles" validate:"max=1024"` - Permissions uint32 `config:"permissions"` - Interval time.Duration `config:"interval"` - RotateOnStartup bool `config:"rotateonstartup"` - RedirectStderr bool `config:"redirect_stderr" yaml:"redirect_stderr"` + Path string `config:"path" yaml:"path"` + Name string `config:"name" yaml:"name"` + MaxSize uint `config:"rotateeverybytes" yaml:"rotateeverybytes" validate:"min=1"` + MaxBackups uint `config:"keepfiles" yaml:"keepfiles" validate:"max=1024"` + Permissions uint32 `config:"permissions"` + Interval time.Duration `config:"interval"` + RotateOnStartup bool `config:"rotateonstartup"` + RedirectStderr bool `config:"redirect_stderr" yaml:"redirect_stderr"` } // MetricsConfig contains configuration used by the monitor to output metrics into the logstream. @@ -76,7 +73,6 @@ func DefaultConfig(environment Environment) Config { return Config{ Level: defaultLevel, Files: FileConfig{ - Suffix: file.SuffixCount, MaxSize: 10 * 1024 * 1024, MaxBackups: 7, Permissions: 0600, diff --git a/libbeat/logp/core.go b/libbeat/logp/core.go index c85a5feeaec6..552c81e92015 100644 --- a/libbeat/logp/core.go +++ b/libbeat/logp/core.go @@ -241,7 +241,6 @@ func makeFileOutput(cfg Config) (zapcore.Core, error) { file.Interval(cfg.Files.Interval), file.RotateOnStartup(cfg.Files.RotateOnStartup), file.RedirectStderr(cfg.Files.RedirectStderr), - file.Suffix(cfg.Files.Suffix), ) if err != nil { return nil, errors.Wrap(err, "failed to create file rotator") diff --git a/libbeat/outputs/fileout/config.go b/libbeat/outputs/fileout/config.go index 28441c93bb02..e80f0a27be90 100644 --- a/libbeat/outputs/fileout/config.go +++ b/libbeat/outputs/fileout/config.go @@ -25,19 +25,17 @@ import ( ) type config struct { - Path string `config:"path"` - Filename string `config:"filename"` - Suffix file.SuffixType `config:"suffix"` - RotateEveryKb uint `config:"rotate_every_kb" validate:"min=1"` - NumberOfFiles uint `config:"number_of_files"` - Codec codec.Config `config:"codec"` - Permissions uint32 `config:"permissions"` - RotateOnStartup bool `config:"rotate_on_startup"` + Path string `config:"path"` + Filename string `config:"filename"` + RotateEveryKb uint `config:"rotate_every_kb" validate:"min=1"` + NumberOfFiles uint `config:"number_of_files"` + Codec codec.Config `config:"codec"` + Permissions uint32 `config:"permissions"` + RotateOnStartup bool `config:"rotate_on_startup"` } func defaultConfig() config { return config{ - Suffix: file.SuffixCount, NumberOfFiles: 7, RotateEveryKb: 10 * 1024, Permissions: 0600, diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index 48a65adc1c0b..b4a10e38396a 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -84,7 +84,6 @@ func (out *fileOutput) init(beat beat.Info, c config) error { var err error out.rotator, err = file.NewFileRotator( path, - file.Suffix(c.Suffix), file.MaxSizeBytes(c.RotateEveryKb*1024), file.MaxBackups(c.NumberOfFiles), file.Permissions(os.FileMode(c.Permissions)), From cd2495931045a71605b77fd69c9120fddabdc352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 11 Nov 2021 09:30:34 +0100 Subject: [PATCH 03/19] init --- libbeat/common/file/rotator_test.go | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/libbeat/common/file/rotator_test.go b/libbeat/common/file/rotator_test.go index 6af7ef625868..cb51b779298c 100644 --- a/libbeat/common/file/rotator_test.go +++ b/libbeat/common/file/rotator_test.go @@ -53,41 +53,37 @@ func TestFileRotatorB(t *testing.T) { } defer r.Close() - expectedPatterns := []string{ - fmt.Sprintf("%s-%s.ndjson", logname, testClock.Now().Format("20060102150405")), - } + firstFile := fmt.Sprintf("%s-%s.ndjson", logname, testClock.Now().Format("20060102150405")) WriteMsg(t, r) - AssertDirContents(t, dir, expectedPatterns...) + AssertDirContents(t, dir, firstFile) testClock.time = time.Date(2021, 11, 11, 11, 13, 0, 0, time.Local) Rotate(t, r) - AssertDirContents(t, dir, expectedPatterns...) + AssertDirContents(t, dir, firstFile) WriteMsg(t, r) - expectedPatterns = append(expectedPatterns, fmt.Sprintf("%s-%s.ndjson", logname, testClock.Now().Format("20060102150405"))) - AssertDirContents(t, dir, expectedPatterns...) + secondFile := fmt.Sprintf("%s-%s.ndjson", logname, testClock.Now().Format("20060102150405")) + AssertDirContents(t, dir, firstFile, secondFile) testClock.time = time.Date(2021, 11, 11, 11, 14, 0, 0, time.Local) Rotate(t, r) - AssertDirContents(t, dir, expectedPatterns...) + AssertDirContents(t, dir, firstFile, secondFile) WriteMsg(t, r) - expectedPatterns = append(expectedPatterns, fmt.Sprintf("%s-%s.ndjson", logname, testClock.Now().Format("20060102150405"))) - AssertDirContents(t, dir, expectedPatterns...) + thirdFile := fmt.Sprintf("%s-%s.ndjson", logname, testClock.Now().Format("20060102150405")) + AssertDirContents(t, dir, firstFile, secondFile, thirdFile) testClock.time = time.Date(2021, 11, 11, 11, 15, 0, 0, time.Local) Rotate(t, r) - expectedPatterns = expectedPatterns[1:] - AssertDirContents(t, dir, expectedPatterns...) + AssertDirContents(t, dir, secondFile, thirdFile) testClock.time = time.Date(2021, 11, 11, 11, 16, 0, 0, time.Local) Rotate(t, r) - expectedPatterns = expectedPatterns[1:] - AssertDirContents(t, dir, expectedPatterns...) + AssertDirContents(t, dir, thirdFile) } func TestFileRotatorConcurrently(t *testing.T) { From 708ad0d5e8d87deac28d94e4139ae823d3629b36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 11 Nov 2021 09:47:44 +0100 Subject: [PATCH 04/19] pass clock to more places --- libbeat/common/file/rotator.go | 4 +-- libbeat/common/file/rotator_test.go | 41 ++++++++++++++++------------- libbeat/common/file/trigger.go | 8 +++--- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/libbeat/common/file/rotator.go b/libbeat/common/file/rotator.go index 2a1be3ced12a..e74eaf0d3511 100644 --- a/libbeat/common/file/rotator.go +++ b/libbeat/common/file/rotator.go @@ -173,7 +173,7 @@ func NewFileRotator(filename string, options ...RotatorOption) (*Rotator, error) shouldRotateOnStart = false } - r.triggers = newTriggers(shouldRotateOnStart, r.interval, r.maxSizeBytes) + r.triggers = newTriggers(shouldRotateOnStart, r.interval, r.maxSizeBytes, r.clock) if r.log != nil { r.log.Debugw("Initialized file rotator", @@ -325,7 +325,7 @@ func (r *Rotator) isRotationTriggered(dataLen uint) (rotateReason, time.Time) { for _, t := range r.triggers { reason := t.TriggerRotation(dataLen) if reason != rotateReasonNoRotate { - return reason, time.Now() + return reason, r.clock.Now() } } return rotateReasonNoRotate, time.Time{} diff --git a/libbeat/common/file/rotator_test.go b/libbeat/common/file/rotator_test.go index cb51b779298c..25b3283eddc0 100644 --- a/libbeat/common/file/rotator_test.go +++ b/libbeat/common/file/rotator_test.go @@ -35,55 +35,55 @@ import ( const logMessage = "Test file rotator.\n" -func TestFileRotatorB(t *testing.T) { +func TestFileRotator(t *testing.T) { logp.TestingSetup() dir := t.TempDir() logname := "sample" - testClock := &testClock{time.Date(2021, 11, 11, 11, 12, 0, 0, time.Local)} + c := &testClock{time.Date(2021, 11, 11, 11, 12, 0, 0, time.Local)} filename := filepath.Join(dir, logname) r, err := file.NewFileRotator(filename, file.MaxBackups(2), file.WithLogger(logp.NewLogger("rotator").With(logp.Namespace("rotator"))), - file.WithClock(testClock), + file.WithClock(c), ) if err != nil { t.Fatal(err) } defer r.Close() - firstFile := fmt.Sprintf("%s-%s.ndjson", logname, testClock.Now().Format("20060102150405")) + firstFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format("20060102150405")) WriteMsg(t, r) AssertDirContents(t, dir, firstFile) - testClock.time = time.Date(2021, 11, 11, 11, 13, 0, 0, time.Local) + c.time = time.Date(2021, 11, 11, 11, 13, 0, 0, time.Local) Rotate(t, r) AssertDirContents(t, dir, firstFile) WriteMsg(t, r) - secondFile := fmt.Sprintf("%s-%s.ndjson", logname, testClock.Now().Format("20060102150405")) + secondFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format("20060102150405")) AssertDirContents(t, dir, firstFile, secondFile) - testClock.time = time.Date(2021, 11, 11, 11, 14, 0, 0, time.Local) + c.time = time.Date(2021, 11, 11, 11, 14, 0, 0, time.Local) Rotate(t, r) AssertDirContents(t, dir, firstFile, secondFile) WriteMsg(t, r) - thirdFile := fmt.Sprintf("%s-%s.ndjson", logname, testClock.Now().Format("20060102150405")) + thirdFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format("20060102150405")) AssertDirContents(t, dir, firstFile, secondFile, thirdFile) - testClock.time = time.Date(2021, 11, 11, 11, 15, 0, 0, time.Local) + c.time = time.Date(2021, 11, 11, 11, 15, 0, 0, time.Local) Rotate(t, r) AssertDirContents(t, dir, secondFile, thirdFile) - testClock.time = time.Date(2021, 11, 11, 11, 16, 0, 0, time.Local) + c.time = time.Date(2021, 11, 11, 11, 16, 0, 0, time.Local) Rotate(t, r) - AssertDirContents(t, dir, thirdFile) + AssertDirContents(t, dir, secondFile, thirdFile) } func TestFileRotatorConcurrently(t *testing.T) { @@ -179,13 +179,15 @@ func TestRotateOnStartup(t *testing.T) { dir := t.TempDir() logname := "rotate_on_open" - filename := filepath.Join(dir, logname) + c := &testClock{time.Date(2021, 11, 11, 11, 12, 0, 0, time.Local)} + firstFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format("20060102150405")) + filename := filepath.Join(dir, firstFile) // Create an existing log file with this name. CreateFile(t, filename) - AssertDirContents(t, dir, logname) + AssertDirContents(t, dir, firstFile) - r, err := file.NewFileRotator(filename, file.RotateOnStartup(false)) + r, err := file.NewFileRotator(filepath.Join(dir, logname), file.RotateOnStartup(false), file.WithClock(c)) if err != nil { t.Fatal(err) } @@ -193,24 +195,27 @@ func TestRotateOnStartup(t *testing.T) { WriteMsg(t, r) // The line should have been appended to the existing file without rotation. - AssertDirContents(t, dir, logname) + AssertDirContents(t, dir, firstFile) // Close the first rotator early (the deferred close will be a no-op if // we haven't hit an error by now), so it can't interfere with the second one. r.Close() // Create a second rotator with the default setting of rotateOnStartup=true - r, err = file.NewFileRotator(filename) + c = &testClock{time.Date(2021, 11, 11, 11, 20, 0, 0, time.Local)} + r, err = file.NewFileRotator(filepath.Join(dir, logname), file.WithClock(c)) if err != nil { t.Fatal(err) } defer r.Close() // The directory contents shouldn't change until the first Write. - AssertDirContents(t, dir, logname) + AssertDirContents(t, dir, firstFile) + + secondFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format("20060102150405")) WriteMsg(t, r) - AssertDirContents(t, dir, logname, logname+".1") + AssertDirContents(t, dir, firstFile, secondFile) } func TestRotateDat(t *testing.T) { diff --git a/libbeat/common/file/trigger.go b/libbeat/common/file/trigger.go index d96d748f9221..22c128a334bb 100644 --- a/libbeat/common/file/trigger.go +++ b/libbeat/common/file/trigger.go @@ -52,14 +52,14 @@ type trigger interface { TriggerRotation(dataLen uint) rotateReason } -func newTriggers(rotateOnStartup bool, interval time.Duration, maxSizeBytes uint) []trigger { +func newTriggers(rotateOnStartup bool, interval time.Duration, maxSizeBytes uint, clock clock) []trigger { triggers := make([]trigger, 0) if rotateOnStartup { triggers = append(triggers, &initTrigger{}) } if interval > 0 { - triggers = append(triggers, newIntervalTrigger(interval)) + triggers = append(triggers, newIntervalTrigger(interval, clock)) } if maxSizeBytes > 0 { triggers = append(triggers, &sizeTrigger{maxSizeBytes: maxSizeBytes, size: 0}) @@ -113,8 +113,8 @@ func (realClock) Now() time.Time { return time.Now() } -func newIntervalTrigger(interval time.Duration) trigger { - t := intervalTrigger{interval: interval, clock: realClock{}} +func newIntervalTrigger(interval time.Duration, clock clock) trigger { + t := intervalTrigger{interval: interval, clock: clock} switch interval { case time.Second: From 9c18b620d203d5b56e219f9385b7883e6d6f720b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 16 Nov 2021 16:41:28 +0100 Subject: [PATCH 05/19] more follow up --- libbeat/common/file/rotator.go | 66 +++++++++- libbeat/common/file/rotator_test.go | 123 +++++++----------- .../elastic-agent/pkg/core/logger/logger.go | 2 - 3 files changed, 108 insertions(+), 83 deletions(-) diff --git a/libbeat/common/file/rotator.go b/libbeat/common/file/rotator.go index e74eaf0d3511..75f57b530a68 100644 --- a/libbeat/common/file/rotator.go +++ b/libbeat/common/file/rotator.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "sync" "time" @@ -32,6 +33,7 @@ const ( // MaxBackupsLimit is the upper bound on the number of backup files. Any values // greater will result in an error. MaxBackupsLimit = 1024 + DateFormat = "2006010215" ) // rotater is the interface responsible for rotating and finding files. @@ -396,7 +398,7 @@ func newDateRotater(log Logger, filename string, clock clock) rotater { clock: clock, filenamePrefix: filename + "-", extension: ".ndjson", - format: "20060102150405", + format: DateFormat, } d.currentFilename = d.filenamePrefix + d.clock.Now().Format(d.format) + d.extension @@ -427,7 +429,22 @@ func (d *dateRotator) Rotate(reason rotateReason, rotateTime time.Time) error { d.log.Debugw("Rotating file", "filename", d.currentFilename, "reason", reason) } - d.currentFilename = d.filenamePrefix + rotateTime.Format(d.format) + d.extension + newFileNamePrefix := d.filenamePrefix + rotateTime.Format(d.format) + files, err := filepath.Glob(newFileNamePrefix + "*" + d.extension) + if err != nil { + return fmt.Errorf("failed to get possible files: %+v", err) + } + + if len(files) == 0 { + d.currentFilename = newFileNamePrefix + d.extension + return nil + } + + d.SortModTimeLogs(files) + order := d.OrderLog(files[len(files)-1]) + + d.currentFilename = newFileNamePrefix + "-" + strconv.Itoa(order.count+1) + d.extension + return nil } @@ -439,10 +456,18 @@ func (d *dateRotator) RotatedFiles() []string { } } + for i, name := range files { + if name == d.ActiveFile() { + files = append(files[:i], files[i+1:]...) + break + } + } + d.SortModTimeLogs(files) return files } +// the newest file is going to be the last func (d *dateRotator) SortModTimeLogs(strings []string) { sort.Slice( strings, @@ -452,10 +477,39 @@ func (d *dateRotator) SortModTimeLogs(strings []string) { ) } -func (d *dateRotator) OrderLog(filename string) time.Time { - ts, err := time.Parse(d.filenamePrefix+d.format, filepath.Base(filename)) +type logOrder struct { + count int + datetime time.Time +} + +func (o logOrder) After(other logOrder) bool { + if o.datetime.Equal(other.datetime) { + return other.count > o.count + } + return !o.datetime.After(other.datetime) +} + +func (d *dateRotator) OrderLog(filename string) logOrder { + prefixLength := len(d.filenamePrefix) + filenameLength := prefixLength + len(d.format) + + var o logOrder + if filenameLength+len(d.extension) < len(filename) { + countStr := filename[filenameLength:] + countStr = countStr[:len(countStr)-len(d.extension)] + if len(countStr) > 0 { + countStr = countStr[1:] + c, err := strconv.Atoi(countStr) + if err != nil { + return o + } + o.count = c + } + } + ts, err := time.Parse(d.format, filename[prefixLength:filenameLength]) if err != nil { - return time.Time{} + return o } - return ts + o.datetime = ts + return o } diff --git a/libbeat/common/file/rotator_test.go b/libbeat/common/file/rotator_test.go index 25b3283eddc0..13e6ce147507 100644 --- a/libbeat/common/file/rotator_test.go +++ b/libbeat/common/file/rotator_test.go @@ -21,8 +21,6 @@ import ( "fmt" "os" "path/filepath" - "regexp" - "sort" "sync" "testing" "time" @@ -40,7 +38,7 @@ func TestFileRotator(t *testing.T) { dir := t.TempDir() logname := "sample" - c := &testClock{time.Date(2021, 11, 11, 11, 12, 0, 0, time.Local)} + c := &testClock{time.Date(2021, 11, 11, 0, 0, 0, 0, time.Local)} filename := filepath.Join(dir, logname) r, err := file.NewFileRotator(filename, @@ -53,35 +51,35 @@ func TestFileRotator(t *testing.T) { } defer r.Close() - firstFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format("20060102150405")) + firstFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) WriteMsg(t, r) AssertDirContents(t, dir, firstFile) - c.time = time.Date(2021, 11, 11, 11, 13, 0, 0, time.Local) + c.time = time.Date(2021, 11, 12, 0, 0, 0, 0, time.Local) Rotate(t, r) AssertDirContents(t, dir, firstFile) WriteMsg(t, r) - secondFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format("20060102150405")) + secondFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) AssertDirContents(t, dir, firstFile, secondFile) - c.time = time.Date(2021, 11, 11, 11, 14, 0, 0, time.Local) + c.time = time.Date(2021, 11, 13, 0, 0, 0, 0, time.Local) Rotate(t, r) AssertDirContents(t, dir, firstFile, secondFile) WriteMsg(t, r) - thirdFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format("20060102150405")) + thirdFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) AssertDirContents(t, dir, firstFile, secondFile, thirdFile) - c.time = time.Date(2021, 11, 11, 11, 15, 0, 0, time.Local) + c.time = time.Date(2021, 11, 14, 0, 0, 0, 0, time.Local) Rotate(t, r) AssertDirContents(t, dir, secondFile, thirdFile) - c.time = time.Date(2021, 11, 11, 11, 16, 0, 0, time.Local) + c.time = time.Date(2021, 11, 15, 0, 0, 0, 0, time.Local) Rotate(t, r) AssertDirContents(t, dir, secondFile, thirdFile) } @@ -111,29 +109,29 @@ func TestDailyRotation(t *testing.T) { dir := t.TempDir() logname := "daily" - dateFormat := "2006-01-02" - today := time.Now().Format(dateFormat) - yesterday := time.Now().AddDate(0, 0, -1).Format(dateFormat) - twoDaysAgo := time.Now().AddDate(0, 0, -2).Format(dateFormat) + yesterday := time.Now().AddDate(0, 0, -1).Format(file.DateFormat) + twoDaysAgo := time.Now().AddDate(0, 0, -2).Format(file.DateFormat) // seed directory with existing log files files := []string{ - logname + "-" + yesterday + "-1", - logname + "-" + yesterday + "-2", - logname + "-" + yesterday + "-3", - logname + "-" + yesterday + "-4", - logname + "-" + yesterday + "-5", - logname + "-" + yesterday + "-6", - logname + "-" + yesterday + "-7", - logname + "-" + yesterday + "-8", - logname + "-" + yesterday + "-9", - logname + "-" + yesterday + "-10", - logname + "-" + yesterday + "-11", - logname + "-" + yesterday + "-12", - logname + "-" + yesterday + "-13", - logname + "-" + twoDaysAgo + "-1", - logname + "-" + twoDaysAgo + "-2", - logname + "-" + twoDaysAgo + "-3", + logname + "-" + twoDaysAgo + ".ndjson", + logname + "-" + twoDaysAgo + "-1.ndjson", + logname + "-" + twoDaysAgo + "-2.ndjson", + logname + "-" + twoDaysAgo + "-3.ndjson", + logname + "-" + yesterday + ".ndjson", + logname + "-" + yesterday + "-1.ndjson", + logname + "-" + yesterday + "-2.ndjson", + logname + "-" + yesterday + "-3.ndjson", + logname + "-" + yesterday + "-4.ndjson", + logname + "-" + yesterday + "-5.ndjson", + logname + "-" + yesterday + "-6.ndjson", + logname + "-" + yesterday + "-7.ndjson", + logname + "-" + yesterday + "-8.ndjson", + logname + "-" + yesterday + "-9.ndjson", + logname + "-" + yesterday + "-10.ndjson", + logname + "-" + yesterday + "-11.ndjson", + logname + "-" + yesterday + "-12.ndjson", + logname + "-" + yesterday + "-13.ndjson", } for _, f := range files { @@ -153,25 +151,26 @@ func TestDailyRotation(t *testing.T) { Rotate(t, r) - AssertDirContents(t, dir, logname+"-"+yesterday+"-12", logname+"-"+yesterday+"-13") + AssertDirContents(t, dir, logname+"-"+yesterday+"-12.ndjson", logname+"-"+yesterday+"-13.ndjson") WriteMsg(t, r) - AssertDirContents(t, dir, logname+"-"+yesterday+"-12", logname+"-"+yesterday+"-13", logname) + today := time.Now().Format(file.DateFormat) + AssertDirContents(t, dir, logname+"-"+yesterday+"-12.ndjson", logname+"-"+yesterday+"-13.ndjson", logname+"-"+today+".ndjson") Rotate(t, r) - AssertDirContents(t, dir, logname+"-"+yesterday+"-13", logname+"-"+today+"-1") + AssertDirContents(t, dir, logname+"-"+yesterday+"-13.ndjson", logname+"-"+today+".ndjson") WriteMsg(t, r) - AssertDirContents(t, dir, logname+"-"+yesterday+"-13", logname+"-"+today+"-1", logname) + AssertDirContents(t, dir, logname+"-"+yesterday+"-13.ndjson", logname+"-"+today+".ndjson", logname+"-"+today+"-1.ndjson") for i := 0; i < (int(maxSizeBytes)/len(logMessage))+1; i++ { WriteMsg(t, r) } - AssertDirContents(t, dir, logname+"-"+today+"-1", logname+"-"+today+"-2", logname) + AssertDirContents(t, dir, logname+"-"+today+"-1.ndjson", logname+"-"+today+"-2.ndjson") } // Tests the FileConfig.RotateOnStartup parameter @@ -179,8 +178,8 @@ func TestRotateOnStartup(t *testing.T) { dir := t.TempDir() logname := "rotate_on_open" - c := &testClock{time.Date(2021, 11, 11, 11, 12, 0, 0, time.Local)} - firstFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format("20060102150405")) + c := &testClock{time.Date(2021, 11, 11, 0, 0, 0, 0, time.Local)} + firstFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) filename := filepath.Join(dir, firstFile) // Create an existing log file with this name. @@ -202,7 +201,7 @@ func TestRotateOnStartup(t *testing.T) { r.Close() // Create a second rotator with the default setting of rotateOnStartup=true - c = &testClock{time.Date(2021, 11, 11, 11, 20, 0, 0, time.Local)} + c = &testClock{time.Date(2021, 11, 12, 0, 0, 0, 0, time.Local)} r, err = file.NewFileRotator(filepath.Join(dir, logname), file.WithClock(c)) if err != nil { t.Fatal(err) @@ -212,7 +211,7 @@ func TestRotateOnStartup(t *testing.T) { // The directory contents shouldn't change until the first Write. AssertDirContents(t, dir, firstFile) - secondFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format("20060102150405")) + secondFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) WriteMsg(t, r) AssertDirContents(t, dir, firstFile, secondFile) @@ -224,7 +223,8 @@ func TestRotateDat(t *testing.T) { logname := "beatname" filename := filepath.Join(dir, logname) - r, err := file.NewFileRotator(filename, file.MaxBackups(1)) + c := &testClock{time.Date(2021, 11, 11, 0, 0, 0, 0, time.Local)} + r, err := file.NewFileRotator(filename, file.MaxBackups(1), file.WithClock(c)) if err != nil { t.Fatal(err) } @@ -232,24 +232,24 @@ func TestRotateDat(t *testing.T) { WriteMsg(t, r) - firstExpectedPattern := fmt.Sprintf("%s-%s.*", logname, time.Now().Format("20060102150405")) - AssertDirContentsPattern(t, dir, firstExpectedPattern) + firstFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) + AssertDirContents(t, dir, firstFile) - time.Sleep(2 * time.Second) - secondExpectedPattern := fmt.Sprintf("%s-%s.*", logname, time.Now().Format("20060102150405")) + c.time = time.Date(2021, 11, 13, 0, 0, 0, 0, time.Local) + secondFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) Rotate(t, r) WriteMsg(t, r) - AssertDirContentsPattern(t, dir, firstExpectedPattern, secondExpectedPattern) + AssertDirContents(t, dir, firstFile, secondFile) - time.Sleep(2 * time.Second) - thirdExpectedPattern := fmt.Sprintf("%s-%s.*", logname, time.Now().Format("20060102150405")) + c.time = time.Date(2021, 11, 15, 0, 0, 0, 0, time.Local) + thirdFile := fmt.Sprintf("%s-%s.ndjson", logname, c.Now().Format(file.DateFormat)) Rotate(t, r) WriteMsg(t, r) - AssertDirContentsPattern(t, dir, secondExpectedPattern, thirdExpectedPattern) + AssertDirContents(t, dir, secondFile, thirdFile) } func CreateFile(t *testing.T, filename string) { @@ -280,33 +280,6 @@ func AssertDirContents(t *testing.T, dir string, files ...string) { assert.ElementsMatch(t, files, names) } -func AssertDirContentsPattern(t *testing.T, dir string, patterns ...string) { - t.Helper() - - f, err := os.Open(dir) - if err != nil { - t.Fatal(err) - } - - names, err := f.Readdirnames(-1) - if err != nil { - t.Fatal(err) - } - if len(patterns) != len(names) { - t.Fatal("unexpected number of files") - } - - sort.Strings(patterns) - sort.Strings(names) - for i := 0; i < len(patterns); i++ { - matches, err := regexp.MatchString(patterns[i], names[i]) - if err != nil { - t.Fatal(err) - } - assert.True(t, matches, "pattern: %s name: %s", patterns[i], names[i]) - } -} - func WriteMsg(t *testing.T, r *file.Rotator) { t.Helper() diff --git a/x-pack/elastic-agent/pkg/core/logger/logger.go b/x-pack/elastic-agent/pkg/core/logger/logger.go index 77cc4260accc..51f23d974c3b 100644 --- a/x-pack/elastic-agent/pkg/core/logger/logger.go +++ b/x-pack/elastic-agent/pkg/core/logger/logger.go @@ -103,7 +103,6 @@ func DefaultLoggingConfig() *Config { cfg.ToFiles = true cfg.Files.Path = paths.Logs() cfg.Files.Name = agentName - cfg.Files.Suffix = file.SuffixDate return &cfg } @@ -124,7 +123,6 @@ func makeInternalFileOutput(cfg *Config) (zapcore.Core, error) { file.Interval(defaultCfg.Files.Interval), file.RotateOnStartup(defaultCfg.Files.RotateOnStartup), file.RedirectStderr(defaultCfg.Files.RedirectStderr), - file.Suffix(cfg.Files.Suffix), ) if err != nil { return nil, errors.New("failed to create internal file rotator") From 66e6f06cd52dd522ecffdafeb2330bfe9974c025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 16 Nov 2021 16:43:30 +0100 Subject: [PATCH 06/19] more --- libbeat/common/file/rotator_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/common/file/rotator_test.go b/libbeat/common/file/rotator_test.go index 13e6ce147507..240198133ebd 100644 --- a/libbeat/common/file/rotator_test.go +++ b/libbeat/common/file/rotator_test.go @@ -170,7 +170,7 @@ func TestDailyRotation(t *testing.T) { WriteMsg(t, r) } - AssertDirContents(t, dir, logname+"-"+today+"-1.ndjson", logname+"-"+today+"-2.ndjson") + AssertDirContents(t, dir, logname+"-"+today+"-1.ndjson", logname+"-"+today+"-2.ndjson", logname+"-"+today+"-3.ndjson") } // Tests the FileConfig.RotateOnStartup parameter @@ -217,7 +217,7 @@ func TestRotateOnStartup(t *testing.T) { AssertDirContents(t, dir, firstFile, secondFile) } -func TestRotateDat(t *testing.T) { +func TestRotate(t *testing.T) { dir := t.TempDir() logname := "beatname" From cf43d55c36d6e4adfc87c82c4e93ae06a8599db2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 16 Nov 2021 16:45:45 +0100 Subject: [PATCH 07/19] less changes --- libbeat/common/file/rotator_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/libbeat/common/file/rotator_test.go b/libbeat/common/file/rotator_test.go index 240198133ebd..aa218318a6b5 100644 --- a/libbeat/common/file/rotator_test.go +++ b/libbeat/common/file/rotator_test.go @@ -114,11 +114,6 @@ func TestDailyRotation(t *testing.T) { // seed directory with existing log files files := []string{ - logname + "-" + twoDaysAgo + ".ndjson", - logname + "-" + twoDaysAgo + "-1.ndjson", - logname + "-" + twoDaysAgo + "-2.ndjson", - logname + "-" + twoDaysAgo + "-3.ndjson", - logname + "-" + yesterday + ".ndjson", logname + "-" + yesterday + "-1.ndjson", logname + "-" + yesterday + "-2.ndjson", logname + "-" + yesterday + "-3.ndjson", @@ -132,6 +127,9 @@ func TestDailyRotation(t *testing.T) { logname + "-" + yesterday + "-11.ndjson", logname + "-" + yesterday + "-12.ndjson", logname + "-" + yesterday + "-13.ndjson", + logname + "-" + twoDaysAgo + "-1.ndjson", + logname + "-" + twoDaysAgo + "-2.ndjson", + logname + "-" + twoDaysAgo + "-3.ndjson", } for _, f := range files { From c22f413bbb58b53435db200edfde3ba1a4de2123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 16 Nov 2021 17:25:47 +0100 Subject: [PATCH 08/19] cleanup --- libbeat/common/file/rotator.go | 67 ++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/libbeat/common/file/rotator.go b/libbeat/common/file/rotator.go index 75f57b530a68..d2d165cf0ff6 100644 --- a/libbeat/common/file/rotator.go +++ b/libbeat/common/file/rotator.go @@ -390,6 +390,13 @@ type dateRotator struct { filenamePrefix string currentFilename string extension string + + prefixLen int + filenameLen int + extensionLen int + + // logOrderCache is used to cache log file meta information between rotations + logOrderCache map[string]logOrder } func newDateRotater(log Logger, filename string, clock clock) rotater { @@ -399,7 +406,11 @@ func newDateRotater(log Logger, filename string, clock clock) rotater { filenamePrefix: filename + "-", extension: ".ndjson", format: DateFormat, + logOrderCache: make(map[string]logOrder), } + d.prefixLen = len(d.filenamePrefix) + d.filenameLen = d.prefixLen + len(DateFormat) + d.extensionLen = len(d.extension) d.currentFilename = d.filenamePrefix + d.clock.Now().Format(d.format) + d.extension files, err := filepath.Glob(d.filenamePrefix + "*" + d.extension) @@ -429,6 +440,8 @@ func (d *dateRotator) Rotate(reason rotateReason, rotateTime time.Time) error { d.log.Debugw("Rotating file", "filename", d.currentFilename, "reason", reason) } + d.logOrderCache = make(map[string]logOrder, 0) + newFileNamePrefix := d.filenamePrefix + rotateTime.Format(d.format) files, err := filepath.Glob(newFileNamePrefix + "*" + d.extension) if err != nil { @@ -443,7 +456,7 @@ func (d *dateRotator) Rotate(reason rotateReason, rotateTime time.Time) error { d.SortModTimeLogs(files) order := d.OrderLog(files[len(files)-1]) - d.currentFilename = newFileNamePrefix + "-" + strconv.Itoa(order.count+1) + d.extension + d.currentFilename = newFileNamePrefix + "-" + strconv.Itoa(order.index+1) + d.extension return nil } @@ -467,7 +480,7 @@ func (d *dateRotator) RotatedFiles() []string { return files } -// the newest file is going to be the last +// SortModTimeLogs puts newest file to the last func (d *dateRotator) SortModTimeLogs(strings []string) { sort.Slice( strings, @@ -477,39 +490,53 @@ func (d *dateRotator) SortModTimeLogs(strings []string) { ) } +// logOrder stores information required to sort log files +// parsed out from the following format {filename}-{datetime}-{index}.ndjson type logOrder struct { - count int + index int datetime time.Time } func (o logOrder) After(other logOrder) bool { if o.datetime.Equal(other.datetime) { - return other.count > o.count + return other.index > o.index } return !o.datetime.After(other.datetime) } func (d *dateRotator) OrderLog(filename string) logOrder { - prefixLength := len(d.filenamePrefix) - filenameLength := prefixLength + len(d.format) + if o, ok := d.logOrderCache[filename]; ok { + return o + } var o logOrder - if filenameLength+len(d.extension) < len(filename) { - countStr := filename[filenameLength:] - countStr = countStr[:len(countStr)-len(d.extension)] - if len(countStr) > 0 { - countStr = countStr[1:] - c, err := strconv.Atoi(countStr) - if err != nil { - return o - } - o.count = c - } - } - ts, err := time.Parse(d.format, filename[prefixLength:filenameLength]) + var err error + + o.datetime, err = time.Parse(d.format, filename[d.prefixLen:d.filenameLen]) if err != nil { return o } - o.datetime = ts + + if d.isFilenameWithIndex(filename) { + o.index, err = d.filenameIndex(filename) + if err != nil { + return o + } + } + + d.logOrderCache[filename] = o + return o } + +func (d *dateRotator) isFilenameWithIndex(filename string) bool { + return d.filenameLen+d.extensionLen < len(filename) +} + +func (d *dateRotator) filenameIndex(filename string) (int, error) { + indexStr := filename[d.filenameLen+1 : len(filename)-d.extensionLen] + if len(indexStr) > 0 { + return strconv.Atoi(indexStr) + } + return 0, nil +} From 99c6f1688b5bb78bf58b63b777419b97c9310343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 16 Nov 2021 17:45:30 +0100 Subject: [PATCH 09/19] update config and docs --- auditbeat/auditbeat.reference.yml | 5 ----- filebeat/filebeat.reference.yml | 5 ----- heartbeat/heartbeat.reference.yml | 5 ----- journalbeat/journalbeat.reference.yml | 5 ----- libbeat/_meta/config/logging.reference.yml.tmpl | 5 ----- libbeat/docs/loggingconfig.asciidoc | 9 --------- metricbeat/metricbeat.reference.yml | 5 ----- packetbeat/packetbeat.reference.yml | 5 ----- winlogbeat/winlogbeat.reference.yml | 5 ----- x-pack/auditbeat/auditbeat.reference.yml | 5 ----- x-pack/filebeat/filebeat.reference.yml | 5 ----- x-pack/functionbeat/functionbeat.reference.yml | 5 ----- x-pack/heartbeat/heartbeat.reference.yml | 5 ----- x-pack/metricbeat/metricbeat.reference.yml | 5 ----- x-pack/osquerybeat/osquerybeat.reference.yml | 5 ----- x-pack/packetbeat/packetbeat.reference.yml | 5 ----- x-pack/winlogbeat/winlogbeat.reference.yml | 5 ----- 17 files changed, 89 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index a6adc66ac343..240196f2b8bc 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -1443,11 +1443,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. auditbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 9ddcc1483290..8633e0125f19 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -2355,11 +2355,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. filebeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index e3b07c015de4..3710a1d4ad86 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1589,11 +1589,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. heartbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Heartbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index f0a85bd09eb0..91bd73fcb127 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -1386,11 +1386,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. journalbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Journalbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/libbeat/_meta/config/logging.reference.yml.tmpl b/libbeat/_meta/config/logging.reference.yml.tmpl index 9a1e91a0af85..f4ca435be017 100644 --- a/libbeat/_meta/config/logging.reference.yml.tmpl +++ b/libbeat/_meta/config/logging.reference.yml.tmpl @@ -67,8 +67,3 @@ logging.files: # Rotate existing logs on startup rather than appending to the existing # file. Defaults to true. # rotateonstartup: true - - # Rotated files are either suffixed with a number e.g. {{.BeatName}}.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count diff --git a/libbeat/docs/loggingconfig.asciidoc b/libbeat/docs/loggingconfig.asciidoc index 4b47c45d5fbd..d6232e2cb07e 100644 --- a/libbeat/docs/loggingconfig.asciidoc +++ b/libbeat/docs/loggingconfig.asciidoc @@ -246,15 +246,6 @@ Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h are boundary-aligned with minutes, hours, days, weeks, months, and years as reported by the local system clock. All other intervals are calculated from the unix epoch. Defaults to disabled. - -[float] -==== `logging.files.suffix` - -When a log rotation happens it can either rename older files with -an incresing index if `count` is configured. The other option is `date` -that appends the current date and time to the end of the filename. -When the log is rotated a new file is created and older files -remain untouched. endif::serverless[] [float] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index a980244e48c8..9d67e32789ba 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -2266,11 +2266,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. metricbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 2875560800ab..5932567bdee3 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1938,11 +1938,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. packetbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Packetbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 3831d25a6fe6..ea047dfb0fe7 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -1366,11 +1366,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. winlogbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 8f77c76ee144..715e6654145b 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -1499,11 +1499,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. auditbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index f45a0ab851d3..079b5be680d7 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -4508,11 +4508,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. filebeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 1c68196633a2..61625bf73d36 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -1237,11 +1237,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. functionbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Functionbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index e3b07c015de4..3710a1d4ad86 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -1589,11 +1589,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. heartbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Heartbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 307683acb1ab..5707b96c85d2 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -2787,11 +2787,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. metricbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index 945cfe9ce785..c702f6eaa660 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -956,11 +956,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. osquerybeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Osquerybeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 2875560800ab..5932567bdee3 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -1938,11 +1938,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. packetbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Packetbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index efe36dec0313..045c273916a4 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -1409,11 +1409,6 @@ logging.files: # file. Defaults to true. # rotateonstartup: true - # Rotated files are either suffixed with a number e.g. winlogbeat.1 when - # renamed during rotation. Or when set to date, the date is added to - # the end of the file. On rotation a new file is created, older files are untouched. - #suffix: count - # ============================= X-Pack Monitoring ============================== # Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The From 6b8bb79e66a3dea8a688aec9f7c0da9589ad9253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 16 Nov 2021 17:46:18 +0100 Subject: [PATCH 10/19] add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b0b60113c7ca..5aebb0071791 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -27,6 +27,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - add_process_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620] - add_docker_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620] - Index template's default_fields setting is only populated with ECS fields. {pull}28596[28596] {issue}28215[28215] +- Remove options `logging.files.suffix` and default to datetime endings. {pull}28927[28927] *Auditbeat* From 724275f49418ab7b36be0716c505bd9fe413484a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 17 Nov 2021 14:03:12 +0100 Subject: [PATCH 11/19] follow up --- libbeat/common/file/rotator_test.go | 2 +- libbeat/logp/core_test.go | 2 +- libbeat/tests/system/beat/beat.py | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/libbeat/common/file/rotator_test.go b/libbeat/common/file/rotator_test.go index aa218318a6b5..6294b042d130 100644 --- a/libbeat/common/file/rotator_test.go +++ b/libbeat/common/file/rotator_test.go @@ -87,7 +87,7 @@ func TestFileRotator(t *testing.T) { func TestFileRotatorConcurrently(t *testing.T) { dir := t.TempDir() - filename := filepath.Join(dir, "sample.log") + filename := filepath.Join(dir, "sample") r, err := file.NewFileRotator(filename, file.MaxBackups(2)) if err != nil { t.Fatal(err) diff --git a/libbeat/logp/core_test.go b/libbeat/logp/core_test.go index 293f57098eb0..f8537eb6aa7b 100644 --- a/libbeat/logp/core_test.go +++ b/libbeat/logp/core_test.go @@ -153,7 +153,7 @@ func TestLoggingECSFields(t *testing.T) { Level: DebugLevel, development: true, Files: FileConfig{ - Name: "beat1.log", + Name: "beat1", }, } ToObserverOutput()(&cfg) diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 195741b1d317..e250c9ab62a0 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -207,7 +207,8 @@ def start_beat(self, config = self.beat_name + ".yml" if output is None: - output = self.beat_name + ".log" + today = datetime.now().strftime("%Y%m%d") + output = self.beat_name+"-"+today+".ndjson" args = [cmd, "-systemTest"] if os.getenv("TEST_COVERAGE") == "true": @@ -368,7 +369,8 @@ def get_log(self, logfile=None): Returns the log as a string. """ if logfile is None: - logfile = self.beat_name + ".log" + today = datetime.now().strftime("%Y%m%d") + logfile = self.beat_name + "-"+today+".ndjson" with open(os.path.join(self.working_dir, logfile), 'r', encoding="utf_8") as f: data = f.read() @@ -416,8 +418,9 @@ def log_contains_count(self, msg, logfile=None, ignore_case=False): msg = msg.lower() # Init defaults + today = datetime.now().strftime("%Y%m%d") if logfile is None: - logfile = self.beat_name + ".log" + logfile = self.beat_name +"-"+today+ ".ndjson" try: with open(os.path.join(self.working_dir, logfile), "r", encoding="utf_8") as f: From b117295e0a795370c242482c3f40fac9f49ab93c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 17 Nov 2021 15:42:14 +0100 Subject: [PATCH 12/19] more more --- libbeat/common/file/rotator.go | 2 +- libbeat/tests/system/base.py | 2 ++ libbeat/tests/system/beat/beat.py | 3 +-- libbeat/tests/system/test_base.py | 3 +-- libbeat/tests/system/test_logging.py | 4 ++-- libbeat/tests/system/test_umask.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libbeat/common/file/rotator.go b/libbeat/common/file/rotator.go index d2d165cf0ff6..e32e317dbfb4 100644 --- a/libbeat/common/file/rotator.go +++ b/libbeat/common/file/rotator.go @@ -33,7 +33,7 @@ const ( // MaxBackupsLimit is the upper bound on the number of backup files. Any values // greater will result in an error. MaxBackupsLimit = 1024 - DateFormat = "2006010215" + DateFormat = "20060102" ) // rotater is the interface responsible for rotating and finding files. diff --git a/libbeat/tests/system/base.py b/libbeat/tests/system/base.py index 7fd9f1dbb490..a768d80e31ea 100644 --- a/libbeat/tests/system/base.py +++ b/libbeat/tests/system/base.py @@ -1,9 +1,11 @@ import os +from datetime import datetime from beat.beat import TestCase from elasticsearch import Elasticsearch, NotFoundError class BaseTest(TestCase): + today = datetime.now().strftime("%Y%m%d") @classmethod def setUpClass(self): diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index e250c9ab62a0..d7c86c66dcf4 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -418,9 +418,8 @@ def log_contains_count(self, msg, logfile=None, ignore_case=False): msg = msg.lower() # Init defaults - today = datetime.now().strftime("%Y%m%d") if logfile is None: - logfile = self.beat_name +"-"+today+ ".ndjson" + logfile = self.beat_name +"-" + self.today + ".ndjson" try: with open(os.path.join(self.working_dir, logfile), "r", encoding="utf_8") as f: diff --git a/libbeat/tests/system/test_base.py b/libbeat/tests/system/test_base.py index 97615e7ec00e..fbb8b324f162 100644 --- a/libbeat/tests/system/test_base.py +++ b/libbeat/tests/system/test_base.py @@ -165,8 +165,7 @@ def run(): # remove log, restart beat and check meta file did not change # and same UUID is used in log output. - - os.remove(os.path.join(self.working_dir, "mockbeat.log")) + os.remove(os.path.join(self.working_dir, "mockbeat-" + self.today + ".ndjson")) meta1 = run() assert self.log_contains("Beat ID: {}".format(meta1["uuid"])) diff --git a/libbeat/tests/system/test_logging.py b/libbeat/tests/system/test_logging.py index 4295bfd1776b..4380e25b55c1 100644 --- a/libbeat/tests/system/test_logging.py +++ b/libbeat/tests/system/test_logging.py @@ -39,11 +39,11 @@ def test_file_default(self): """ self.run_beat_with_args("Mockbeat is alive!", logging_args=[]) - self.assert_contains_ecs_log(logfile="logs/mockbeat") + self.assert_contains_ecs_log(logfile="logs/mockbeat-"+self.today+".ndjson") def test_file_ecs(self): """ logs to file with ECS format """ self.run_beat_with_args("Mockbeat is alive!") - self.assert_contains_ecs_log(logfile="logs/mockbeat") + self.assert_contains_ecs_log(logfile="logs/mockbeat-"+self.today+".ndjson") diff --git a/libbeat/tests/system/test_umask.py b/libbeat/tests/system/test_umask.py index e78214e2d9c3..d6a7b7f391c1 100644 --- a/libbeat/tests/system/test_umask.py +++ b/libbeat/tests/system/test_umask.py @@ -30,7 +30,7 @@ def test_output_file_perms(self): """ Test that output file permissions respect default umask """ - output_file_path = os.path.join(self.working_dir, "output", "mockbeat") + output_file_path = os.path.join(self.working_dir, "output", "mockbeat-" + self.today + ".ndjson") perms = stat.S_IMODE(os.lstat(output_file_path).st_mode) self.assertEqual(perms, self.output_file_permissions & ~TestUmask.DEFAULT_UMASK) From 81b1ce8be38cd3cf0c20877bb9f53b58e279647a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 17 Nov 2021 15:43:36 +0100 Subject: [PATCH 13/19] more --- libbeat/tests/system/beat/beat.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index d7c86c66dcf4..0409f4750589 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -207,8 +207,7 @@ def start_beat(self, config = self.beat_name + ".yml" if output is None: - today = datetime.now().strftime("%Y%m%d") - output = self.beat_name+"-"+today+".ndjson" + output = self.beat_name + "-" + self.today + ".ndjson" args = [cmd, "-systemTest"] if os.getenv("TEST_COVERAGE") == "true": @@ -369,8 +368,7 @@ def get_log(self, logfile=None): Returns the log as a string. """ if logfile is None: - today = datetime.now().strftime("%Y%m%d") - logfile = self.beat_name + "-"+today+".ndjson" + logfile = self.beat_name + "-" + self.today + ".ndjson" with open(os.path.join(self.working_dir, logfile), 'r', encoding="utf_8") as f: data = f.read() @@ -404,6 +402,7 @@ def log_contains(self, msg, logfile=None, ignore_case=False): Returns true if the give logfile contains the given message. Note that the msg must be present in a single line. """ + print("bu") return self.log_contains_count(msg, logfile, ignore_case=ignore_case) > 0 @@ -419,8 +418,9 @@ def log_contains_count(self, msg, logfile=None, ignore_case=False): # Init defaults if logfile is None: - logfile = self.beat_name +"-" + self.today + ".ndjson" + logfile = self.beat_name + "-" + self.today + ".ndjson" + print("logfile", logfile, self.working_dir) try: with open(os.path.join(self.working_dir, logfile), "r", encoding="utf_8") as f: for line in f: @@ -432,7 +432,8 @@ def log_contains_count(self, msg, logfile=None, ignore_case=False): line = line.lower() if line.find(msg) >= 0: counter = counter + 1 - except IOError: + except IOError as e: + print(e) counter = -1 return counter @@ -464,7 +465,7 @@ def log_contains_countmap(self, pattern, capture_group, logfile=None): def output_lines(self, output_file=None): """ Count number of lines in a file.""" if output_file is None: - output_file = "output/" + self.beat_name + output_file = "output/" + self.beat_name + "-" + self.today + ".ndjson" try: with open(os.path.join(self.working_dir, output_file), "r", encoding="utf_8") as f: From 4124effe1d862238cf1d62fb62afa248983dde79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 17 Nov 2021 16:49:57 +0100 Subject: [PATCH 14/19] add missing attribute --- libbeat/tests/system/beat/beat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 0409f4750589..96c828d4c370 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -119,6 +119,7 @@ def __del__(self): class TestCase(unittest.TestCase, ComposeMixin): + today = datetime.now().strftime("%Y%m%d") @classmethod def setUpClass(self): From 038061b9a0d6b46db1b21a054920f9bbcd91acc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 18 Nov 2021 11:25:32 +0100 Subject: [PATCH 15/19] adjust paths for test checks --- libbeat/tests/system/beat/beat.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 96c828d4c370..aa34c14a3775 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -265,7 +265,7 @@ def read_output(self, # Init defaults if output_file is None: - output_file = "output/" + self.beat_name + output_file = "output/" + self.beat_name + "-" + self.today + ".ndjson" jsons = [] with open(os.path.join(self.working_dir, output_file), "r", encoding="utf_8") as f: @@ -289,7 +289,7 @@ def read_output_json(self, output_file=None): # Init defaults if output_file is None: - output_file = "output/" + self.beat_name + output_file = "output/" + self.beat_name + "-" + self.today + ".ndjson" jsons = [] with open(os.path.join(self.working_dir, output_file), "r", encoding="utf_8") as f: @@ -381,7 +381,7 @@ def get_log_lines(self, logfile=None): Returns the log lines as a list of strings """ if logfile is None: - logfile = self.beat_name + ".log" + logfile = self.beat_name + "-" + self.today + ".ndjson" with open(os.path.join(self.working_dir, logfile), 'r', encoding="utf_8") as f: data = f.readlines() @@ -446,7 +446,7 @@ def log_contains_countmap(self, pattern, capture_group, logfile=None): counts = {} if logfile is None: - logfile = self.beat_name + ".log" + logfile = self.beat_name + "-" + self.today + ".ndjson" try: with open(os.path.join(self.working_dir, logfile), "r", encoding="utf_8") as f: @@ -481,7 +481,7 @@ def output_has(self, lines, output_file=None): # Init defaults if output_file is None: - output_file = "output/" + self.beat_name + output_file = "output/" + self.beat_name + "-" + self.today + ".ndjson" try: with open(os.path.join(self.working_dir, output_file, ), "r", encoding="utf_8") as f: @@ -496,7 +496,7 @@ def output_is_empty(self, output_file=None): # Init defaults if output_file is None: - output_file = "output/" + self.beat_name + output_file = "output/" + self.beat_name + "-" + self.today + ".ndjson" try: with open(os.path.join(self.working_dir, output_file, ), "r", encoding="utf_8") as f: @@ -660,7 +660,7 @@ def output_count(self, pred, output_file=None): # Init defaults if output_file is None: - output_file = "output/" + self.beat_name + output_file = "output/" + self.beat_name + "-" + self.today + ".ndjson" try: with open(os.path.join(self.working_dir, output_file), "r", encoding="utf_8") as f: From 3d087d478b7a92432e295d0b2be6ca969837144a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 18 Nov 2021 13:44:17 +0100 Subject: [PATCH 16/19] more follow up --- filebeat/tests/system/filebeat.py | 2 +- filebeat/tests/system/test_harvester.py | 4 ++-- filebeat/tests/system/test_input.py | 6 +++--- filebeat/tests/system/test_multiline.py | 8 ++++---- filebeat/tests/system/test_registrar.py | 4 ++-- libbeat/tests/system/beat/beat.py | 1 - packetbeat/tests/system/packetbeat.py | 11 +++++++++-- 7 files changed, 21 insertions(+), 15 deletions(-) diff --git a/filebeat/tests/system/filebeat.py b/filebeat/tests/system/filebeat.py index 92df15911bb8..9f1f7905d162 100644 --- a/filebeat/tests/system/filebeat.py +++ b/filebeat/tests/system/filebeat.py @@ -36,7 +36,7 @@ def access_registry(self, name=None, data_path=None): return Registry(data_path, name) def log_access(self, file=None): - file = file if file else self.beat_name + ".log" + file = file if file else self.beat_name + "-" + self.today + ".ndjson" return LogState(os.path.join(self.working_dir, file)) def has_registry(self, name=None, data_path=None): diff --git a/filebeat/tests/system/test_harvester.py b/filebeat/tests/system/test_harvester.py index f56a51d0fe07..ecbb23fab77f 100644 --- a/filebeat/tests/system/test_harvester.py +++ b/filebeat/tests/system/test_harvester.py @@ -493,11 +493,11 @@ def test_boms(self, fb_encoding, py_encoding, bom): filebeat = self.start_beat(output=fb_encoding + ".log") self.wait_until( - lambda: self.output_has(lines=1, output_file="output/" + fb_encoding), + lambda: self.output_has(lines=1, output_file="output/" + fb_encoding + "-" + self.today + ".ndjson"), max_timeout=10) # Verify that output does not contain bom - output = self.read_output_json(output_file="output/" + fb_encoding) + output = self.read_output_json(output_file="output/" + fb_encoding + "-" + self.today + ".ndjson") assert output[0]["message"] == message filebeat.kill_and_wait() diff --git a/filebeat/tests/system/test_input.py b/filebeat/tests/system/test_input.py index f9dbd138a2e2..a4215587735c 100644 --- a/filebeat/tests/system/test_input.py +++ b/filebeat/tests/system/test_input.py @@ -633,10 +633,10 @@ def test_restart_recursive_glob(self): with open(testfile_path, 'a') as testfile: testfile.write("entry2\n") - filebeat = self.start_beat(output="filebeat2.log") + filebeat = self.start_beat() self.wait_until( - lambda: self.output_has_message("entry2"), + lambda: self.output_has_message("entry2", output_file="output/filebeat-"+self.today+"-1.ndjson"), max_timeout=10, name="output contains 'entry2'") @@ -783,7 +783,7 @@ def test_inode_marker_based_identity_tracking_to_path_based(self): proc = self.start_beat() # on startup output is rotated - self.wait_until(lambda: self.output_has(lines=1, output_file="output/filebeat.1")) + self.wait_until(lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson")) self.wait_until(lambda: self.output_has(lines=1)) proc.check_kill_and_wait() diff --git a/filebeat/tests/system/test_multiline.py b/filebeat/tests/system/test_multiline.py index ee5384ef5efe..31d4f0258ca1 100644 --- a/filebeat/tests/system/test_multiline.py +++ b/filebeat/tests/system/test_multiline.py @@ -148,11 +148,11 @@ def test_max_lines(self): # Checks line 3 is sent assert True == self.log_contains( - "MetaDataMappingService.java:388", "output/filebeat") + "MetaDataMappingService.java:388", "output/filebeat-" + self.today + ".ndjson") # Checks line 4 is not sent anymore assert False == self.log_contains( - "InternalClusterService.java:388", "output/filebeat") + "InternalClusterService.java:388", "output/filebeat-" + self.today + ".ndjson") # Check that output file has the same number of lines as the log file assert 20 == len(output) @@ -231,10 +231,10 @@ def test_max_bytes(self): output = self.read_output() # Check that first 60 chars are sent - assert True == self.log_contains("cluster.metadata", "output/filebeat") + assert True == self.log_contains("cluster.metadata", "output/filebeat-" + self.today + ".ndjson") # Checks that chars afterwards are not sent - assert False == self.log_contains("Zach", "output/filebeat") + assert False == self.log_contains("Zach", "output/filebeat-" + self.today + ".ndjson") # Check that output file has the same number of lines as the log file assert 20 == len(output) diff --git a/filebeat/tests/system/test_registrar.py b/filebeat/tests/system/test_registrar.py index 02d07a32446a..7e9fa5a6ed49 100644 --- a/filebeat/tests/system/test_registrar.py +++ b/filebeat/tests/system/test_registrar.py @@ -486,7 +486,7 @@ def test_restart_continue(self): # Output file was rotated self.wait_until( - lambda: self.output_has(lines=1, output_file="output/filebeat.1"), + lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson"), max_timeout=10) self.wait_until( @@ -596,7 +596,7 @@ def test_rotating_file_with_restart(self): # Output file was rotated self.wait_until( - lambda: self.output_has(lines=2, output_file="output/filebeat.1"), + lambda: self.output_has(lines=2, output_file="output/filebeat-" + self.today + "-1.ndjson"), max_timeout=10) self.wait_until( diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index aa34c14a3775..e7a2c60640aa 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -403,7 +403,6 @@ def log_contains(self, msg, logfile=None, ignore_case=False): Returns true if the give logfile contains the given message. Note that the msg must be present in a single line. """ - print("bu") return self.log_contains_count(msg, logfile, ignore_case=ignore_case) > 0 diff --git a/packetbeat/tests/system/packetbeat.py b/packetbeat/tests/system/packetbeat.py index 32a9366ee339..1e7205c0b938 100644 --- a/packetbeat/tests/system/packetbeat.py +++ b/packetbeat/tests/system/packetbeat.py @@ -33,7 +33,7 @@ def setUpClass(self): def run_packetbeat(self, pcap, cmd=None, config="packetbeat.yml", - output="packetbeat.log", + output=None, extra_args=[], debug_selectors=[], exit_code=0, @@ -44,6 +44,9 @@ def run_packetbeat(self, pcap, the caller. """ + if output is None: + output = "packetbeat-" + self.today + ".ndjson" + if cmd is None: cmd = self.beat_path + "/packetbeat.test" @@ -119,9 +122,13 @@ def start_packetbeat(self, return proc def read_output(self, - output_file="output/packetbeat", + output_file=None, types=None, required_fields=None): + + if output_file is None: + output_file = "output/packetbeat-"+self.today+".ndjson" + jsons = [] with open(os.path.join(self.working_dir, output_file), "r", encoding='utf_8') as f: for line in f: From dd9b89eb8d9f8565955f3f0aae27bcd35043c0a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 18 Nov 2021 18:41:00 +0100 Subject: [PATCH 17/19] more test fixes --- filebeat/tests/system/test_load.py | 2 +- filebeat/tests/system/test_registrar.py | 10 +++++----- packetbeat/tests/system/packetbeat.py | 5 ++++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/filebeat/tests/system/test_load.py b/filebeat/tests/system/test_load.py index 6a35a6ce2509..b48b83e2659c 100644 --- a/filebeat/tests/system/test_load.py +++ b/filebeat/tests/system/test_load.py @@ -72,7 +72,7 @@ def test_no_missing_events(self): entry_list = [] - with open(self.working_dir + "/output/filebeat") as f: + with open(self.working_dir + "/output/filebeat-" + self.today + ".ndjson") as f: for line in f: content = json.loads(line) v = int(content["message"]) diff --git a/filebeat/tests/system/test_registrar.py b/filebeat/tests/system/test_registrar.py index 7e9fa5a6ed49..29f21dcf86cb 100644 --- a/filebeat/tests/system/test_registrar.py +++ b/filebeat/tests/system/test_registrar.py @@ -486,11 +486,11 @@ def test_restart_continue(self): # Output file was rotated self.wait_until( - lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson"), + lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + ".ndjson"), max_timeout=10) self.wait_until( - lambda: self.output_has(lines=1), + lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson"), max_timeout=10) filebeat.check_kill_and_wait() @@ -505,7 +505,7 @@ def test_restart_continue(self): # should never have been detected assert len(data) == 1 - output = self.read_output() + output = self.read_output(output_file="output/filebeat-" + self.today + "-1.ndjson") # Check that output file has the same number of lines as the log file assert len(output) == 1 @@ -951,8 +951,8 @@ def test_restart_state(self): clean_inactive="3s", ) - filebeat = self.start_beat(output="filebeat2.log") - logs = self.log_access("filebeat2.log") + filebeat = self.start_beat(output="filebeat-"+self.today+"-1.ndjson") + logs = self.log_access("filebeat-"+self.today+"-1.ndjson") # Write additional file for name in restart_files: diff --git a/packetbeat/tests/system/packetbeat.py b/packetbeat/tests/system/packetbeat.py index 1e7205c0b938..21f45be7a00b 100644 --- a/packetbeat/tests/system/packetbeat.py +++ b/packetbeat/tests/system/packetbeat.py @@ -90,7 +90,7 @@ def run_packetbeat(self, pcap, def start_packetbeat(self, cmd=None, config="packetbeat.yml", - output="packetbeat.log", + output=None, extra_args=[], debug_selectors=[]): """ @@ -98,6 +98,9 @@ def start_packetbeat(self, caller is responsible for stopping / waiting for the Proc instance. """ + if output is None: + output = "packetbeat-" + self.today + ".ndjson" + if cmd is None: cmd = self.beat_path + "/packetbeat.test" From d68d4d57c90aeca11a60e4234fc21693902e1c6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 19 Nov 2021 13:46:21 +0100 Subject: [PATCH 18/19] final fixes --- filebeat/tests/system/test_registrar.py | 10 +++++----- packetbeat/tests/system/packetbeat.py | 1 + packetbeat/tests/system/test_0002_thrift_basics.py | 2 +- packetbeat/tests/system/test_0006_wsgi.py | 4 ++-- winlogbeat/tests/system/test_config.py | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/filebeat/tests/system/test_registrar.py b/filebeat/tests/system/test_registrar.py index 29f21dcf86cb..bb55ef547b3e 100644 --- a/filebeat/tests/system/test_registrar.py +++ b/filebeat/tests/system/test_registrar.py @@ -592,15 +592,15 @@ def test_rotating_file_with_restart(self): with open(testfile_path, 'w') as testfile: testfile.write("entry3\n") - filebeat = self.start_beat(output="filebeat2.log") + filebeat = self.start_beat() # Output file was rotated self.wait_until( - lambda: self.output_has(lines=2, output_file="output/filebeat-" + self.today + "-1.ndjson"), + lambda: self.output_has(lines=2), max_timeout=10) self.wait_until( - lambda: self.output_has(lines=1), + lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson"), max_timeout=10) filebeat.check_kill_and_wait() @@ -951,8 +951,8 @@ def test_restart_state(self): clean_inactive="3s", ) - filebeat = self.start_beat(output="filebeat-"+self.today+"-1.ndjson") - logs = self.log_access("filebeat-"+self.today+"-1.ndjson") + filebeat = self.start_beat() + logs = self.log_access() # Write additional file for name in restart_files: diff --git a/packetbeat/tests/system/packetbeat.py b/packetbeat/tests/system/packetbeat.py index 21f45be7a00b..c4727d131837 100644 --- a/packetbeat/tests/system/packetbeat.py +++ b/packetbeat/tests/system/packetbeat.py @@ -131,6 +131,7 @@ def read_output(self, if output_file is None: output_file = "output/packetbeat-"+self.today+".ndjson" + print(output_file) jsons = [] with open(os.path.join(self.working_dir, output_file), "r", encoding='utf_8') as f: diff --git a/packetbeat/tests/system/test_0002_thrift_basics.py b/packetbeat/tests/system/test_0002_thrift_basics.py index 6e0154a437ce..a1a66742bdcf 100644 --- a/packetbeat/tests/system/test_0002_thrift_basics.py +++ b/packetbeat/tests/system/test_0002_thrift_basics.py @@ -208,7 +208,7 @@ def test_thrift_send_request_response(self): self.run_packetbeat(pcap="thrift_integration.pcap", debug_selectors=["thrift"]) - objs = self.read_output() + objs = self.read_output(output_file="output/packetbeat-" + self.today + "-1.ndjson") assert all(["request" not in o for o in objs]) assert all(["response" not in o for o in objs]) diff --git a/packetbeat/tests/system/test_0006_wsgi.py b/packetbeat/tests/system/test_0006_wsgi.py index 6f905942449d..02e52b885a36 100644 --- a/packetbeat/tests/system/test_0006_wsgi.py +++ b/packetbeat/tests/system/test_0006_wsgi.py @@ -100,7 +100,7 @@ def test_send_headers_options(self): ) self.run_packetbeat(pcap="wsgi_loopback.pcap") - objs = self.read_output() + objs = self.read_output(output_file="output/packetbeat-" + self.today + "-1.ndjson") assert len(objs) == 1 o = objs[0] @@ -119,7 +119,7 @@ def test_send_headers_options(self): ) self.run_packetbeat(pcap="wsgi_loopback.pcap") - objs = self.read_output() + objs = self.read_output(output_file="output/packetbeat-" + self.today + "-2.ndjson") assert len(objs) == 1 o = objs[0] diff --git a/winlogbeat/tests/system/test_config.py b/winlogbeat/tests/system/test_config.py index 306f6cf38ef8..aefab4344ba8 100644 --- a/winlogbeat/tests/system/test_config.py +++ b/winlogbeat/tests/system/test_config.py @@ -82,7 +82,7 @@ def run_config_tst(self, pcap=None, exit_code=0): args.extend(["test", "config"]) - output = "winlogbeat.log" + output = "winlogbeat-" + self.today + ".ndjson" with open(os.path.join(self.working_dir, output), "wb") as outfile: proc = subprocess.Popen(args, From 2ff01b3cf13849181b53133ed7fae48d792ee075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 19 Nov 2021 15:17:17 +0100 Subject: [PATCH 19/19] adjust more things again --- winlogbeat/tests/system/test_wineventlog.py | 2 +- x-pack/elastic-agent/pkg/agent/operation/monitoring.go | 6 ++---- x-pack/elastic-agent/pkg/core/logger/logger.go | 2 +- .../pkg/core/monitoring/beats/beats_monitor.go | 2 +- .../elastic-agent/pkg/core/monitoring/beats/monitoring.go | 4 ++-- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/winlogbeat/tests/system/test_wineventlog.py b/winlogbeat/tests/system/test_wineventlog.py index 347b8585d059..9bbf5a7ded43 100644 --- a/winlogbeat/tests/system/test_wineventlog.py +++ b/winlogbeat/tests/system/test_wineventlog.py @@ -52,7 +52,7 @@ def test_resume_reading_events(self): # remove the output file, otherwise there is a race condition # in read_events() below where it reads the results of the previous # execution - os.unlink(os.path.join(self.working_dir, "output", self.beat_name)) + os.unlink(os.path.join(self.working_dir, "output", self.beat_name + "-" + self.today + ".ndjson")) msg = "Second event" self.write_event_log(msg) diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index dedec28762af..19cd40cda913 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -234,10 +234,8 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa }, }, "paths": []string{ - filepath.Join(paths.Home(), "logs", "elastic-agent-json.log"), - filepath.Join(paths.Home(), "logs", "elastic-agent-json.log*"), - filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log"), - filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log*"), + filepath.Join(paths.Home(), "logs", "elastic-agent-*.ndjson"), + filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-*.ndjson"), }, "index": fmt.Sprintf("logs-elastic_agent-%s", monitoringNamespace), "processors": []map[string]interface{}{ diff --git a/x-pack/elastic-agent/pkg/core/logger/logger.go b/x-pack/elastic-agent/pkg/core/logger/logger.go index 51f23d974c3b..a63bc94f9de1 100644 --- a/x-pack/elastic-agent/pkg/core/logger/logger.go +++ b/x-pack/elastic-agent/pkg/core/logger/logger.go @@ -114,7 +114,7 @@ func makeInternalFileOutput(cfg *Config) (zapcore.Core, error) { // defaultCfg is used to set the defaults for the file rotation of the internal logging // these settings cannot be changed by a user configuration defaultCfg := logp.DefaultConfig(logp.DefaultEnvironment) - filename := filepath.Join(paths.Home(), "logs", fmt.Sprintf("%s-json.log", cfg.Beat)) + filename := filepath.Join(paths.Home(), "logs", cfg.Beat) rotator, err := file.NewFileRotator(filename, file.MaxSizeBytes(defaultCfg.Files.MaxSize), diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go index f96f5ecf9164..e00860ec1f10 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go @@ -131,7 +131,7 @@ func (b *Monitor) EnrichArgs(spec program.Spec, pipelineID string, args []string if isSidecar { logFile += "_monitor" } - logFile = fmt.Sprintf("%s-json.log", logFile) + logFile = fmt.Sprintf("%s", logFile) appendix = append(appendix, "-E", "logging.files.path="+loggingPath, "-E", "logging.files.name="+logFile, diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go index 240ce5adbb22..d56aee0ef2b3 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go @@ -16,9 +16,9 @@ import ( const ( // args: data path, pipeline name, application name - logFileFormat = "%s/logs/%s/%s-json.log" + logFileFormat = "%s/logs/%s/%s" // args: data path, install path, pipeline name, application name - logFileFormatWin = "%s\\logs\\%s\\%s-json.log" + logFileFormatWin = "%s\\logs\\%s\\%s" // args: pipeline name, application name mbEndpointFileFormatWin = `npipe:///%s-%s`