diff --git a/logger/logger.go b/logger/logger.go index b89e33c70a..6f6c0f3627 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -4,10 +4,6 @@ package logger import ( - "io" - "os" - "time" - "github.com/influxdata/wlog" "go.uber.org/zap" "go.uber.org/zap/buffer" @@ -96,28 +92,6 @@ func ConvertToLetterLevel(l zapcore.Level) string { return string(l.CapitalString()[0]) } -func SampledLogger() *zap.Logger { - stdout := zapcore.AddSync(os.Stdout) - - level := zap.NewAtomicLevelAt(zap.InfoLevel) - productionCfg := newProductionEncoderConfig() - productionCfg.TimeKey = "timestamp" - productionCfg.EncodeTime = zapcore.ISO8601TimeEncoder - jsonEncoder := zapcore.NewJSONEncoder(productionCfg) - - jsonOutCore := zapcore.NewCore(jsonEncoder, stdout, level) - - // Create a logger that samples every Nth message after the first M messages every S seconds - // where N = sc.Thereafter, M = sc.Initial, S = sc.Tick. - samplingCore := zapcore.NewSamplerWithOptions( - jsonOutCore, - time.Hour, // interval - 5, // log first 3 entries - 500, // thereafter log zero entires within the interval - ) - return zap.New(samplingCore) -} func init() { loggerLevel = zap.NewAtomicLevelAt(zapcore.InfoLevel) } - diff --git a/plugins/outputs/cloudwatchlogs/pusher.go b/plugins/outputs/cloudwatchlogs/pusher.go index c214f59b7d..916fdb4d90 100644 --- a/plugins/outputs/cloudwatchlogs/pusher.go +++ b/plugins/outputs/cloudwatchlogs/pusher.go @@ -4,10 +4,8 @@ package cloudwatchlogs import ( - cwaLogger "github.com/aws/amazon-cloudwatch-agent/logger" "math/rand" "sort" - "strconv" "sync" "time" @@ -18,14 +16,13 @@ import ( "github.com/aws/amazon-cloudwatch-agent/logs" "github.com/aws/amazon-cloudwatch-agent/profiler" - - "go.uber.org/zap" ) const ( reqSizeLimit = 1024 * 1024 reqEventsLimit = 10000 warnOldTimeStamp = 1 * 24 * time.Hour + logWarnInterval = 1 * 5 * time.Minute ) var ( @@ -56,6 +53,7 @@ type pusher struct { sequenceToken *string lastValidTime int64 lastUpdateTime time.Time + lastWarnMessage time.Time needSort bool stop <-chan struct{} lastSentTime time.Time @@ -133,8 +131,6 @@ func hasValidTime(e logs.LogEvent) bool { func (p *pusher) start() { defer p.wg.Done() - sampledLogger := cwaLogger.SampledLogger() - ec := make(chan logs.LogEvent) // Merge events from both blocking and non-blocking channel @@ -160,7 +156,7 @@ func (p *pusher) start() { p.resetFlushTimer() } - ce := p.convertEvent(e, sampledLogger) + ce := p.convertEvent(e) et := time.Unix(*ce.Timestamp/1000, *ce.Timestamp%1000) // Cloudwatch Log Timestamp is in Millisecond // A batch of log events in a single request cannot span more than 24 hours. @@ -416,7 +412,7 @@ func (p *pusher) resetFlushTimer() { p.flushTimer.Reset(p.FlushTimeout) } -func (p *pusher) convertEvent(e logs.LogEvent, sampledLogger *zap.Logger) *cloudwatchlogs.InputLogEvent { +func (p *pusher) convertEvent(e logs.LogEvent) *cloudwatchlogs.InputLogEvent { message := e.Message() if len(message) > msgSizeLimit { @@ -432,7 +428,12 @@ func (p *pusher) convertEvent(e logs.LogEvent, sampledLogger *zap.Logger) *cloud if !p.lastUpdateTime.IsZero() { // Check when timestamp has an interval of 1 days. if time.Since(p.lastUpdateTime) > warnOldTimeStamp { - sampledLogger.Warn("Unable to parse timestamp, using last valid timestamp found in the logs " + strconv.Itoa(int(p.lastValidTime)) + ": which is at least older than 1 day for log group " + p.Group + ": ") + { + if time.Since(p.lastWarnMessage) > logWarnInterval { + p.Log.Warnf("Unable to parse timestamp, using last valid timestamp found in the logs %v: which is at least older than 1 day for log group %v: ", p.lastValidTime, p.Group) + p.lastWarnMessage = time.Now() + } + } } } } else { @@ -442,6 +443,7 @@ func (p *pusher) convertEvent(e logs.LogEvent, sampledLogger *zap.Logger) *cloud t = e.Time().UnixNano() / 1000000 p.lastValidTime = t p.lastUpdateTime = time.Now() + p.lastWarnMessage = time.Time{} } return &cloudwatchlogs.InputLogEvent{ Message: &message, @@ -467,4 +469,3 @@ func (inputLogEvents ByTimestamp) Swap(i, j int) { func (inputLogEvents ByTimestamp) Less(i, j int) bool { return *inputLogEvents[i].Timestamp < *inputLogEvents[j].Timestamp } -