Add structure logging to Vitess

go.mod

@@ -113,6 +113,9 @@ require (
 	github.com/bndr/gotabulate v1.1.2
 	github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249
 	github.com/openark/golib v0.0.0-20210531070646-355f37940af8
+	github.com/planetscale/log v0.0.0-20221118170849-fb599bc35c50
+	github.com/slok/noglog v0.2.0
+	go.uber.org/zap v1.23.0
 	golang.org/x/exp v0.0.0-20221114191408-850992195362
 )
 
@@ -185,7 +188,6 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	go.uber.org/zap v1.23.0 // indirect
 	go4.org/intern v0.0.0-20220617035311-6925f38cc365 // indirect
 	go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20221114191408-850992195362 // indirect
@@ -204,3 +206,5 @@ require (
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )
+
+replace github.com/google/glog => github.com/planetscale/noglog v0.2.1-0.20210421230640-bea75fcd2e8e

go.sum

@@ -650,6 +650,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
+github.com/planetscale/log v0.0.0-20221118170849-fb599bc35c50 h1:yf0iVpE57riOj2+cLr6Fy6qX6eCLc5RLK3Kc70apP6c=
+github.com/planetscale/log v0.0.0-20221118170849-fb599bc35c50/go.mod h1:SE8Q9QtLD8tfq8bM7rGLJnnWfmxt6mTXGkfGbft1vJI=
 github.com/planetscale/pargzip v0.0.0-20201116224723-90c7fc03ea8a h1:y0OpQ4+5tKxeh9+H+2cVgASl9yMZYV9CILinKOiKafA=
 github.com/planetscale/pargzip v0.0.0-20201116224723-90c7fc03ea8a/go.mod h1:GJFUzQuXIoB2Kjn1ZfDhJr/42D5nWOqRcIQVgCxTuIE=
 github.com/planetscale/vtprotobuf v0.3.0 h1:oMrOdDFHS1ADc0dHtC2EApxiM5xd0cQkZeibm0WgXiQ=
@@ -713,6 +715,8 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sjmudd/stopwatch v0.1.1 h1:x45OvxFB5OtCkjvYtzRF5fWB857Jzjjk84Oyd5C5ebw=
 github.com/sjmudd/stopwatch v0.1.1/go.mod h1:BLw0oIQJ1YLXBO/q9ufK/SgnKBVIkC2qrm6uy78Zw6U=
+github.com/slok/noglog v0.2.0 h1:1czu4l2EoJ8L92UwdSXXa1Y+c5TIjFAFm2P+mjej95E=
+github.com/slok/noglog v0.2.0/go.mod h1:TfKxwpEZPT+UA83bQ6RME146k0MM4e8mwHLf6bhcGDI=
 github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=

go/vt/logutil/planetscale_logger.go

@@ -0,0 +1,47 @@
+package logutil
+
+import (
+	pslog "github.com/planetscale/log"
+	noglog "github.com/slok/noglog"
+
+	"vitess.io/vitess/go/vt/log"
+)
+
+type PSLogger pslog.SugaredLogger
+
+// SetPlanetScaleLogger in-place noglog replacement with PlanetScales' logger.
+func SetPlanetScaleLogger(conf *pslog.Config) (psLogger *pslog.SugaredLogger) {
+	// Use the passed configuration instead of the default configuration
+	if conf != nil {
+		configLogger, _ := conf.Build()
+		psLogger = configLogger.Sugar()
+	} else {
+		psLogger = pslog.NewPlanetScaleSugarLogger()
+	}
+
+	noglog.SetLogger(&noglog.LoggerFunc{
+		DebugfFunc: func(f string, a ...interface{}) { psLogger.Debugf(f, a...) },
+		InfofFunc:  func(f string, a ...interface{}) { psLogger.Infof(f, a...) },
+		WarnfFunc:  func(f string, a ...interface{}) { psLogger.Warnf(f, a...) },
+		ErrorfFunc: func(f string, a ...interface{}) { psLogger.Errorf(f, a...) },
+	})
+
+	log.Flush = noglog.Flush
+	log.Info = noglog.Info
+	log.Infof = noglog.Infof
+	log.InfoDepth = noglog.InfoDepth
+	log.Warning = noglog.Warning
+	log.Warningf = noglog.Warningf
+	log.WarningDepth = noglog.WarningDepth
+	log.Error = noglog.Error
+	log.Errorf = noglog.Errorf
+	log.ErrorDepth = noglog.ErrorDepth
+	log.Exit = noglog.Exit
+	log.Exitf = noglog.Exitf
+	log.ExitDepth = noglog.ExitDepth
+	log.Fatal = noglog.Fatal
+	log.Fatalf = noglog.Fatalf
+	log.FatalDepth = noglog.FatalDepth
+
+	return
+}

go/vt/logutil/planetscale_logger_test.go

@@ -0,0 +1,112 @@
+package logutil
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	pslog "github.com/planetscale/log"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+
+	vtlog "vitess.io/vitess/go/vt/log"
+)
+
+// MemorySink implements zap.Sink by writing all messages to a buffer.
+// It's used to capture the logs.
+type MemorySink struct {
+	*bytes.Buffer
+}
+
+// Implement Close and Sync as no-ops to satisfy the interface. The Write
+// method is provided by the embedded buffer.
+func (s *MemorySink) Close() error { return nil }
+func (s *MemorySink) Sync() error  { return nil }
+
+func SetupLoggerWithMemSink() (sink *MemorySink, err error) {
+	// Create a sink instance, and register it with zap for the "memory"
+	// protocol.
+	sink = &MemorySink{new(bytes.Buffer)}
+	err = zap.RegisterSink("memory", func(*url.URL) (zap.Sink, error) {
+		return sink, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	testLoggerConf := pslog.NewPlanetScaleConfig(pslog.DetectEncoding(), pslog.InfoLevel)
+	testLoggerConf.OutputPaths = []string{"memory://"}
+	testLoggerConf.ErrorOutputPaths = []string{"memory://"}
+	SetPlanetScaleLogger(&testLoggerConf)
+
+	return
+}
+
+func TestPSLogger_Replacing_glog(t *testing.T) {
+	type logMsg struct {
+		Level string `json:"level"`
+		Msg   string `json:"msg"`
+	}
+
+	type testCase struct {
+		name     string
+		logLevel zapcore.Level
+	}
+
+	// Given
+	dummyLogMessage := "testing log"
+	testCases := []testCase{
+		{"log info", pslog.InfoLevel},
+		{"log warn", pslog.WarnLevel},
+		{"log error", pslog.ErrorLevel},
+	}
+
+	sink, err := SetupLoggerWithMemSink()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// When
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			var loggingFunc func(format string, args ...interface{})
+			var expectedLevel string
+
+			switch tc.logLevel {
+			case zapcore.InfoLevel:
+				{
+					loggingFunc = vtlog.Infof
+					expectedLevel = "info"
+				}
+			case zapcore.ErrorLevel:
+				{
+					loggingFunc = vtlog.Errorf
+					expectedLevel = "error"
+				}
+			case zapcore.WarnLevel:
+				{
+					loggingFunc = vtlog.Warningf
+					expectedLevel = "warn"
+				}
+			}
+
+			loggingFunc(dummyLogMessage)
+			// Unmarshal the captured log. This means we're getting a struct log.
+			actualLog := logMsg{}
+			err = json.Unmarshal(sink.Bytes(), &actualLog)
+			if err != nil {
+				t.Error(err)
+			}
+			// Reset the sink so that it'll contain one log per test case.
+			sink.Reset()
+
+			// Then
+			assert.Equal(t, expectedLevel, actualLog.Level)
+			assert.Equal(t, dummyLogMessage, actualLog.Msg)
+
+		})
+	}
+}

go/vt/servenv/servenv.go

@@ -82,6 +82,7 @@ var (
 	onCloseTimeout = time.Nanosecond
 	catchSigpipe   bool
 	maxStackSize   = 64 * 1024 * 1024
+	usePSLogger    bool
 )
 
 // RegisterFlags installs the flags used by Init, Run, and RunDefault.
@@ -98,6 +99,8 @@ func RegisterFlags() {
 
 		// pid_file.go
 		fs.StringVar(&pidFile, "pid_file", pidFile, "If set, the process will write its pid to the named file, and delete it on graceful shutdown.")
+		// Logging
+		fs.BoolVar(&usePSLogger, "structure_logging", usePSLogger, "whether to use structure logging (Planet Scale Log) logger or the original (glog) logger")
 	})
 }
 
@@ -322,6 +325,11 @@ func ParseFlags(cmd string) {
 		os.Exit(0)
 	}
 
+	if usePSLogger {
+		// Replace glog logger with PlanetScale logger
+		logutil.SetPlanetScaleLogger(nil)
+	}
+
 	args := fs.Args()
 	if len(args) > 0 {
 		_flag.Usage()