From ec328a74551d76abdaaf13cf624fcb3e301a2c38 Mon Sep 17 00:00:00 2001 From: alyakimenko Date: Sat, 4 Dec 2021 19:39:02 +0200 Subject: [PATCH] add support for local files to --log-ouput Closes #2249 --- cmd/root.go | 30 +++++--- log/file.go | 162 ++++++++++++++++++++++++++++++++++++++++ log/file_test.go | 168 ++++++++++++++++++++++++++++++++++++++++++ log/levels.go | 40 ++++++++++ log/levels_test.go | 81 ++++++++++++++++++++ log/loki.go | 14 +--- log/tokenizer.go | 2 +- log/tokenizer_test.go | 3 + 8 files changed, 476 insertions(+), 24 deletions(-) create mode 100644 log/file.go create mode 100644 log/file_test.go create mode 100644 log/levels.go create mode 100644 log/levels_test.go diff --git a/cmd/root.go b/cmd/root.go index a4d25573515b..b3c33e49235a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -246,7 +246,7 @@ func (c *rootCommand) rootCmdPersistentFlagSet() *pflag.FlagSet { flags.BoolVarP(&c.commandFlags.quiet, "quiet", "q", false, "disable progress updates") flags.BoolVar(&c.commandFlags.noColor, "no-color", false, "disable colored output") flags.StringVar(&c.logOutput, "log-output", "stderr", - "change the output for k6 logs, possible values are stderr,stdout,none,loki[=host:port]") + "change the output for k6 logs, possible values are stderr,stdout,none,loki[=host:port],file[=./path.fileformat]") flags.StringVar(&c.logFmt, "logformat", "", "log output format") // TODO rename to log-format and warn on old usage flags.StringVarP(&c.commandFlags.address, "address", "a", "localhost:6565", "address for the api server") @@ -279,27 +279,37 @@ func (c *rootCommand) setupLoggers() (<-chan struct{}, error) { } loggerForceColors := false // disable color by default - switch c.logOutput { - case "stderr": + switch line := c.logOutput; { + case line == "stderr": loggerForceColors = !c.commandFlags.noColor && c.commandFlags.stderrTTY c.logger.SetOutput(c.commandFlags.stderr) - case "stdout": + case line == "stdout": loggerForceColors = !c.commandFlags.noColor && c.commandFlags.stdoutTTY c.logger.SetOutput(c.commandFlags.stdout) - case "none": + case line == "none": c.logger.SetOutput(ioutil.Discard) - default: - if !strings.HasPrefix(c.logOutput, "loki") { - return nil, fmt.Errorf("unsupported log output `%s`", c.logOutput) - } + + case strings.HasPrefix(line, "loki"): ch = make(chan struct{}) - hook, err := log.LokiFromConfigLine(c.ctx, c.fallbackLogger, c.logOutput, ch) + hook, err := log.LokiFromConfigLine(c.ctx, c.fallbackLogger, line, ch) if err != nil { return nil, err } c.logger.AddHook(hook) c.logger.SetOutput(ioutil.Discard) // don't output to anywhere else c.logFmt = "raw" + + case strings.HasPrefix(line, "file"): + hook, err := log.FileHookFromConfigLine(c.ctx, c.fallbackLogger, line) + if err != nil { + return nil, err + } + + c.logger.AddHook(hook) + c.logger.SetOutput(ioutil.Discard) + + default: + return nil, fmt.Errorf("unsupported log output `%s`", line) } switch c.logFmt { diff --git a/log/file.go b/log/file.go new file mode 100644 index 000000000000..b77264bcb5de --- /dev/null +++ b/log/file.go @@ -0,0 +1,162 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2020 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +// Package log implements various logrus hooks. +package log + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/sirupsen/logrus" +) + +// fileHookBufferSize is a default size for the fileHook's loglines channel. +const fileHookBufferSize = 100 + +// fileHook is a hook to handle writing to local files. +type fileHook struct { + fallbackLogger logrus.FieldLogger + loglines chan []byte + path string + w io.WriteCloser + bw *bufio.Writer + levels []logrus.Level +} + +// FileHookFromConfigLine returns new fileHook hook. +func FileHookFromConfigLine( + ctx context.Context, fallbackLogger logrus.FieldLogger, line string, +) (logrus.Hook, error) { + hook := &fileHook{ + fallbackLogger: fallbackLogger, + levels: logrus.AllLevels, + } + + parts := strings.SplitN(line, "=", 2) + if parts[0] != "file" { + return nil, fmt.Errorf("logfile configuration should be in the form `file=path-to-local-file` but is `%s`", line) + } + + if err := hook.parseArgs(line); err != nil { + return nil, err + } + + if err := hook.openFile(); err != nil { + return nil, err + } + + hook.loglines = hook.loop(ctx) + + return hook, nil +} + +func (h *fileHook) parseArgs(line string) error { + tokens, err := tokenize(line) + if err != nil { + return fmt.Errorf("error while parsing logfile configuration %w", err) + } + + for _, token := range tokens { + switch token.key { + case "file": + if token.value == "" { + return fmt.Errorf("filepath must not be empty") + } + h.path = token.value + case "level": + h.levels, err = parseLevels(token.value) + if err != nil { + return err + } + default: + return fmt.Errorf("unknown logfile config key %s", token.key) + } + } + + return nil +} + +// openFile opens logfile and initializes writers. +func (h *fileHook) openFile() error { + if _, err := os.Stat(filepath.Dir(h.path)); os.IsNotExist(err) { + return fmt.Errorf("provided directory '%s' does not exist", filepath.Dir(h.path)) + } + + file, err := os.OpenFile(h.path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o600) + if err != nil { + return fmt.Errorf("failed to open logfile %s: %w", h.path, err) + } + + h.w = file + h.bw = bufio.NewWriter(file) + + return nil +} + +func (h *fileHook) loop(ctx context.Context) chan []byte { + loglines := make(chan []byte, fileHookBufferSize) + + go func() { + defer close(loglines) + + for { + select { + case entry := <-loglines: + if _, err := h.bw.Write(entry); err != nil { + h.fallbackLogger.Errorf("failed to write a log message to a logfile: %w", err) + } + case <-ctx.Done(): + if err := h.bw.Flush(); err != nil { + h.fallbackLogger.Errorf("failed to flush buffer: %w", err) + } + + if err := h.w.Close(); err != nil { + h.fallbackLogger.Errorf("failed to close logfile: %w", err) + } + + return + } + } + }() + + return loglines +} + +// Fire writes the log file to defined path. +func (h *fileHook) Fire(entry *logrus.Entry) error { + message, err := entry.Bytes() + if err != nil { + return fmt.Errorf("failed to get a log entry bytes: %w", err) + } + + h.loglines <- message + return nil +} + +// Levels returns configured log levels. +func (h *fileHook) Levels() []logrus.Level { + return h.levels +} diff --git a/log/file_test.go b/log/file_test.go new file mode 100644 index 000000000000..59da6a5d4e2a --- /dev/null +++ b/log/file_test.go @@ -0,0 +1,168 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2020 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package log + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "os" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type nopCloser struct { + io.Writer + closed chan struct{} +} + +func (nc *nopCloser) Close() error { + nc.closed <- struct{}{} + return nil +} + +func TestFileHookFromConfigLine(t *testing.T) { + t.Parallel() + + tests := [...]struct { + line string + err bool + errMessage string + res fileHook + }{ + { + line: "file", + err: true, + res: fileHook{ + levels: logrus.AllLevels, + }, + }, + { + line: fmt.Sprintf("file=%s/k6.log,level=info", os.TempDir()), + err: false, + res: fileHook{ + path: fmt.Sprintf("%s/k6.log", os.TempDir()), + levels: logrus.AllLevels[:5], + }, + }, + { + line: "file=./", + err: true, + }, + { + line: "file=/a/c/", + err: true, + }, + { + line: "file=,level=info", + err: true, + errMessage: "filepath must not be empty", + }, + { + line: "file=/tmp/k6.log,level=tea", + err: true, + }, + { + line: "file=/tmp/k6.log,unknown", + err: true, + }, + { + line: "file=/tmp/k6.log,level=", + err: true, + }, + { + line: "file=/tmp/k6.log,level=,", + err: true, + }, + { + line: "file=/tmp/k6.log,unknown=something", + err: true, + errMessage: "unknown logfile config key unknown", + }, + { + line: "unknown=something", + err: true, + errMessage: "logfile configuration should be in the form `file=path-to-local-file` but is `unknown=something`", + }, + } + + for _, test := range tests { + test := test + t.Run(test.line, func(t *testing.T) { + t.Parallel() + + res, err := FileHookFromConfigLine(context.Background(), logrus.New(), test.line) + + if test.err { + require.Error(t, err) + + if test.errMessage != "" { + require.Equal(t, test.errMessage, err.Error()) + } + + return + } + + require.NoError(t, err) + assert.NotNil(t, res.(*fileHook).w) + }) + } +} + +func TestFileHookFire(t *testing.T) { + t.Parallel() + + var buffer bytes.Buffer + nc := &nopCloser{ + Writer: &buffer, + closed: make(chan struct{}), + } + + hook := &fileHook{ + loglines: make(chan []byte), + w: nc, + bw: bufio.NewWriter(nc), + levels: logrus.AllLevels, + } + + ctx, cancel := context.WithCancel(context.Background()) + + hook.loglines = hook.loop(ctx) + + logger := logrus.New() + logger.AddHook(hook) + logger.SetOutput(io.Discard) + + logger.Info("example log line") + + time.Sleep(10 * time.Millisecond) + + cancel() + <-nc.closed + + assert.Contains(t, buffer.String(), "example log line") +} diff --git a/log/levels.go b/log/levels.go new file mode 100644 index 000000000000..e7ea1ce9a844 --- /dev/null +++ b/log/levels.go @@ -0,0 +1,40 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2020 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package log + +import ( + "fmt" + "sort" + + "github.com/sirupsen/logrus" +) + +func parseLevels(level string) ([]logrus.Level, error) { + lvl, err := logrus.ParseLevel(level) + if err != nil { + return nil, fmt.Errorf("unknown log level %s", level) // specifically use a custom error + } + index := sort.Search(len(logrus.AllLevels), func(i int) bool { + return logrus.AllLevels[i] > lvl + }) + + return logrus.AllLevels[:index], nil +} diff --git a/log/levels_test.go b/log/levels_test.go new file mode 100644 index 000000000000..668a87e2ef23 --- /dev/null +++ b/log/levels_test.go @@ -0,0 +1,81 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2020 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package log + +import ( + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func Test_getLevels(t *testing.T) { + t.Parallel() + + tests := [...]struct { + level string + err bool + levels []logrus.Level + }{ + { + level: "info", + err: false, + levels: []logrus.Level{ + logrus.PanicLevel, + logrus.FatalLevel, + logrus.ErrorLevel, + logrus.WarnLevel, + logrus.InfoLevel, + }, + }, + { + level: "error", + err: false, + levels: []logrus.Level{ + logrus.PanicLevel, + logrus.FatalLevel, + logrus.ErrorLevel, + }, + }, + { + level: "tea", + err: true, + levels: nil, + }, + } + + for _, test := range tests { + test := test + t.Run(test.level, func(t *testing.T) { + t.Parallel() + + levels, err := parseLevels(test.level) + + if test.err { + require.Error(t, err) + return + } + + require.NoError(t, err) + require.Equal(t, test.levels, levels) + }) + } +} diff --git a/log/loki.go b/log/loki.go index bd6f9e8cb041..402c127bd889 100644 --- a/log/loki.go +++ b/log/loki.go @@ -144,7 +144,7 @@ func (h *lokiHook) parseArgs(line string) error { return fmt.Errorf("loki msgMaxSize needs to be a positive number, is %d", h.msgMaxSize) } case "level": - h.levels, err = getLevels(value) + h.levels, err = parseLevels(value) if err != nil { return err } @@ -165,18 +165,6 @@ func (h *lokiHook) parseArgs(line string) error { return nil } -func getLevels(level string) ([]logrus.Level, error) { - lvl, err := logrus.ParseLevel(level) - if err != nil { - return nil, fmt.Errorf("unknown log level %s", level) // specifically use a custom error - } - index := sort.Search(len(logrus.AllLevels), func(i int) bool { - return logrus.AllLevels[i] > lvl - }) - - return logrus.AllLevels[:index], nil -} - // fill one of two equally sized slices with entries and then push it while filling the other one // TODO benchmark this //nolint:funlen diff --git a/log/tokenizer.go b/log/tokenizer.go index 53c89253fb36..e59298f22da1 100644 --- a/log/tokenizer.go +++ b/log/tokenizer.go @@ -36,7 +36,7 @@ type tokenizer struct { func (t *tokenizer) readKey() (string, error) { start := t.i for ; t.i < len(t.s); t.i++ { - if t.s[t.i] == '=' { + if t.s[t.i] == '=' && t.i != len(t.s)-1 { t.i++ return t.s[start : t.i-1], nil diff --git a/log/tokenizer_test.go b/log/tokenizer_test.go index fa67f8b29c20..bdf499cdc632 100644 --- a/log/tokenizer_test.go +++ b/log/tokenizer_test.go @@ -61,4 +61,7 @@ func TestTokenizer(t *testing.T) { }, }, tokens) assert.NoError(t, err) + + _, err = tokenize("empty=") + assert.EqualError(t, err, "key `empty=` with no value") }