From da8969c30a3d539a1178215b9c194cfe7b05ae59 Mon Sep 17 00:00:00 2001 From: Yuwen Ma Date: Tue, 9 Apr 2019 15:48:46 -0700 Subject: [PATCH] Add a log-file-max-size flag to fix large logfile (>1.8GB) truncation. --- klog.go | 28 ++++++++++++++++++++++++++-- klog_file.go | 2 +- klog_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 76 insertions(+), 5 deletions(-) diff --git a/klog.go b/klog.go index 69e6e1768..7c6d54f78 100644 --- a/klog.go +++ b/klog.go @@ -78,6 +78,7 @@ import ( "fmt" "io" stdLog "log" + "math" "os" "path/filepath" "runtime" @@ -410,6 +411,9 @@ func InitFlags(flagset *flag.FlagSet) { } flagset.StringVar(&logging.logDir, "log_dir", "", "If non-empty, write log files in this directory") flagset.StringVar(&logging.logFile, "log_file", "", "If non-empty, use this log file") + flagset.Uint64Var(&logging.logFileMaxSize, "log_file_max_size", 0, + "Defines the maximum size a log file can grow to. Unit is megabytes. "+ + "If the value is 0, the maximum file size is unlimited.") flagset.BoolVar(&logging.toStderr, "logtostderr", true, "log to standard error instead of files") flagset.BoolVar(&logging.alsoToStderr, "alsologtostderr", false, "log to standard error as well as files") flagset.Var(&logging.verbosity, "v", "number for the log level verbosity") @@ -472,6 +476,10 @@ type loggingT struct { // with the log-dir option. logFile string + // When logFile is specified, this limiter makes sure the logFile won't exceeds a certain size. When exceeds, the + // logFile will be cleaned up. If this value is 0, no size limitation will be applied to logFile. + logFileMaxSize uint64 + // If true, do not add the prefix headers, useful when used with SetOutput skipHeaders bool @@ -874,8 +882,24 @@ func (sb *syncBuffer) Sync() error { return sb.file.Sync() } +var realMaxSizeCalc = func () uint64 { + if logging.logFile != "" { + if logging.logFileMaxSize == 0 { + // If logFileMaxSize is zero, we don't have limitations on the log size. + return math.MaxUint64 + } else { + // Flag logFileMaxSize is in MB for user convenience. + return logging.logFileMaxSize * 1024 * 1024 + } + } + // If "log_file" flag is not specified, the target file (sb.file) will be cleaned up when reaches a fixed size. + return MaxSize +} + +var PrecomputedMaxSize = realMaxSizeCalc() + func (sb *syncBuffer) Write(p []byte) (n int, err error) { - if sb.nbytes+uint64(len(p)) >= MaxSize { + if sb.nbytes+uint64(len(p)) >= PrecomputedMaxSize { if err := sb.rotateFile(time.Now(), false); err != nil { sb.logger.exit(err) } @@ -890,7 +914,7 @@ func (sb *syncBuffer) Write(p []byte) (n int, err error) { // rotateFile closes the syncBuffer's file and starts a new one. // The startup argument indicates whether this is the initial startup of klog. -// If startup is true, existing files are opened for apending instead of truncated. +// If startup is true, existing files are opened for appending instead of truncated. func (sb *syncBuffer) rotateFile(now time.Time, startup bool) error { if sb.file != nil { sb.Flush() diff --git a/klog_file.go b/klog_file.go index 1c4897f4f..e4010ad4d 100644 --- a/klog_file.go +++ b/klog_file.go @@ -98,7 +98,7 @@ var onceLogDirs sync.Once // successfully, create also attempts to update the symlink for that tag, ignoring // errors. // The startup argument indicates whether this is the initial startup of klog. -// If startup is true, existing files are opened for apending instead of truncated. +// If startup is true, existing files are opened for appending instead of truncated. func create(tag string, t time.Time, startup bool) (f *os.File, filename string, err error) { if logging.logFile != "" { f, err := openOrCreate(logging.logFile, startup) diff --git a/klog_test.go b/klog_test.go index b5f25c328..f8bfd75c2 100644 --- a/klog_test.go +++ b/klog_test.go @@ -339,7 +339,7 @@ func TestRollover(t *testing.T) { } defer func(previous uint64) { MaxSize = previous }(MaxSize) MaxSize = 512 - + PrecomputedMaxSize = realMaxSizeCalc() Info("x") // Be sure we have a file. info, ok := logging.file[infoLog].(*syncBuffer) if !ok { @@ -370,7 +370,7 @@ func TestRollover(t *testing.T) { if fname0 == fname1 { t.Errorf("info.f.Name did not change: %v", fname0) } - if info.nbytes >= MaxSize { + if info.nbytes >= PrecomputedMaxSize { t.Errorf("file size was not reset: %d", info.nbytes) } } @@ -487,3 +487,50 @@ func BenchmarkHeader(b *testing.B) { logging.putBuffer(buf) } } + +// Test the logic on checking log size limitation. +func TestFileSizeCheck(t *testing.T) { + setFlags() + testData := map[string]struct { + testLogFile string + testLogFileMaxSize uint64 + testCurrentSize uint64 + expectedResult bool + }{ + "logFile not specified, exceeds max size": { + testLogFile: "", + testLogFileMaxSize: 1, + testCurrentSize: 1024 * 1024 * 2000, //exceeds the maxSize + expectedResult: true, + }, + + "logFile not specified, not exceeds max size": { + testLogFile: "", + testLogFileMaxSize: 1, + testCurrentSize: 1024 * 1024 * 1000, //smaller than the maxSize + expectedResult: false, + }, + "logFile specified, exceeds max size": { + testLogFile: "/tmp/test.log", + testLogFileMaxSize: 500, // 500MB + testCurrentSize: 1024 * 1024 * 1000, //exceeds the logFileMaxSize + expectedResult: true, + }, + "logFile specified, not exceeds max size": { + testLogFile: "/tmp/test.log", + testLogFileMaxSize: 500, // 500MB + testCurrentSize: 1024 * 1024 * 300, //smaller than the logFileMaxSize + expectedResult: false, + }, + } + + for name, test := range testData { + logging.logFile = test.testLogFile + logging.logFileMaxSize = test.testLogFileMaxSize + actualResult := test.testCurrentSize >= realMaxSizeCalc() + if test.expectedResult != actualResult { + t.Fatalf("Error on test case '%v': Was expecting result equals %v, got %v", + name, test.expectedResult, actualResult) + } + } +}