Move log tokenizer to lib/strvals package (#3446)

Up until now the tokenizer implementation was only used to parse the log
output option configuration. With our current intention of using this
format for other options, such as traces output, it becomes evident that
this implementation should be moved away from the log package and into a
more generic one.

lib/strvals/doc.go

@@ -0,0 +1,3 @@
+// Package strvals provides parsing utilities for strval lines, which follow
+// the format: name=value,topname.subname=value.
+package strvals

log/tokenizer.go → lib/strvals/parser.go

@@ -1,9 +1,10 @@
-package log
+package strvals
 
 import "fmt"
 
-type token struct {
-	key, value string
+// Token represents a key-value token.
+type Token struct {
+	Key, Value string
 	inside     rune // shows whether it's inside a given collection, currently [ means it's an array
 }
 
@@ -64,8 +65,10 @@ func (t *tokenizer) readArray() (string, error) {
 	return t.s[start:], fmt.Errorf("array value for key `%s` didn't end", t.currentKey)
 }
 
-func tokenize(s string) ([]token, error) {
-	result := []token{}
+// Parse parses the input string into key-value tokens following strvals format:
+// name=value,topname.subname=value.
+func Parse(s string) ([]Token, error) {
+	result := []Token{}
 	t := &tokenizer{s: s}
 
 	var err error
@@ -79,19 +82,19 @@ func tokenize(s string) ([]token, error) {
 			t.i++
 			value, err = t.readArray()
 
-			result = append(result, token{
-				key:    t.currentKey,
-				value:  value,
+			result = append(result, Token{
+				Key:    t.currentKey,
+				Value:  value,
 				inside: '[',
 			})
 			if err != nil {
 				return result, err
 			}
 		} else {
 			value = t.readValue()
-			result = append(result, token{
-				key:   t.currentKey,
-				value: value,
+			result = append(result, Token{
+				Key:   t.currentKey,
+				Value: value,
 			})
 		}
 	}

log/tokenizer_test.go → lib/strvals/parser_test.go

@@ -1,4 +1,4 @@
-package log
+package strvals
 
 import (
 	"testing"
@@ -7,26 +7,26 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestTokenizer(t *testing.T) {
+func TestParser(t *testing.T) {
 	t.Parallel()
 
 	input := "loki=something,s.e=2231,s=12,12=3,a=[1,2,3],b=[1],s=c"
-	tokens, err := tokenize(input)
+	tokens, err := Parse(input)
 	require.NoError(t, err)
 
-	expected := []token{
-		{key: "loki", value: "something"},
-		{key: "s.e", value: "2231"},
-		{key: "s", value: "12"},
-		{key: "12", value: "3"},
-		{key: "a", value: "1,2,3", inside: '['},
-		{key: "b", value: "1", inside: '['},
-		{key: "s", value: "c"},
+	expected := []Token{
+		{Key: "loki", Value: "something"},
+		{Key: "s.e", Value: "2231"},
+		{Key: "s", Value: "12"},
+		{Key: "12", Value: "3"},
+		{Key: "a", Value: "1,2,3", inside: '['},
+		{Key: "b", Value: "1", inside: '['},
+		{Key: "s", Value: "c"},
 	}
 	assert.Equal(t, expected, tokens)
 }
 
-func TestTokenizerInvalid(t *testing.T) {
+func TestParserInvalid(t *testing.T) {
 	t.Parallel()
 
 	tests := []struct {
@@ -61,7 +61,7 @@ func TestTokenizerInvalid(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			t.Parallel()
 
-			_, err := tokenize(test.input)
+			_, err := Parse(test.input)
 			require.EqualError(t, err, test.errorMsg)
 		})
 	}

log/file.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"go.k6.io/k6/lib/fsext"
+	"go.k6.io/k6/lib/strvals"
 )
 
 // fileHookBufferSize is a default size for the fileHook's loglines channel.
@@ -55,25 +56,25 @@ func FileHookFromConfigLine(
 }
 
 func (h *fileHook) parseArgs(line string) error {
-	tokens, err := tokenize(line)
+	tokens, err := strvals.Parse(line)
 	if err != nil {
 		return fmt.Errorf("error while parsing logfile configuration %w", err)
 	}
 
 	for _, token := range tokens {
-		switch token.key {
+		switch token.Key {
 		case "file":
-			if token.value == "" {
+			if token.Value == "" {
 				return fmt.Errorf("filepath must not be empty")
 			}
-			h.path = token.value
+			h.path = token.Value
 		case "level":
-			h.levels, err = parseLevels(token.value)
+			h.levels, err = parseLevels(token.Value)
 			if err != nil {
 				return err
 			}
 		default:
-			return fmt.Errorf("unknown logfile config key %s", token.key)
+			return fmt.Errorf("unknown logfile config key %s", token.Key)
 		}
 	}
 

log/loki.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/sirupsen/logrus"
+	"go.k6.io/k6/lib/strvals"
 )
 
 // lokiHook is a Logrus hook for flushing to Loki.
@@ -77,14 +78,14 @@ func LokiFromConfigLine(fallbackLogger logrus.FieldLogger, line string) (AsyncHo
 }
 
 func (h *lokiHook) parseArgs(line string) error {
-	tokens, err := tokenize(line)
+	tokens, err := strvals.Parse(line)
 	if err != nil {
 		return fmt.Errorf("error while parsing loki configuration %w", err)
 	}
 
 	for _, token := range tokens {
-		key := token.key
-		value := token.value
+		key := token.Key
+		value := token.Value
 
 		var err error
 		switch key {