Enable logging in ES client

pkg/es/config/config.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"crypto/tls"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -32,6 +33,7 @@ import (
 	"github.com/olivere/elastic"
 	"github.com/uber/jaeger-lib/metrics"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapgrpc"
 
 	"github.com/jaegertracing/jaeger/pkg/config/tlscfg"
 	"github.com/jaegertracing/jaeger/pkg/es"
@@ -67,6 +69,7 @@ type Configuration struct {
 	CreateIndexTemplates  bool           `mapstructure:"create_mappings"`
 	UseILM                bool           `mapstructure:"use_ilm"`
 	Version               uint           `mapstructure:"version"`
+	LogLevel              string         `mapstructure:"log_level"`
 }
 
 // TagsAsFields holds configuration for tag schema.
@@ -102,6 +105,7 @@ type ClientBuilder interface {
 	GetVersion() uint
 	TagKeysAsFields() ([]string, error)
 	GetUseILM() bool
+	GetLogLevel() string
 }
 
 // NewClient creates a new ElasticSearch client
@@ -237,6 +241,9 @@ func (c *Configuration) ApplyDefaults(source *Configuration) {
 	if c.MaxDocCount == 0 {
 		c.MaxDocCount = source.MaxDocCount
 	}
+	if c.LogLevel == "" {
+		c.LogLevel = source.LogLevel
+	}
 }
 
 // GetNumShards returns number of shards from Configuration
@@ -300,6 +307,11 @@ func (c *Configuration) GetUseILM() bool {
 	return c.UseILM
 }
 
+// GetLogLevel returns the log-level the ES client should log at.
+func (c *Configuration) GetLogLevel() string {
+	return c.LogLevel
+}
+
 // GetTokenFilePath returns file path containing the bearer token
 func (c *Configuration) GetTokenFilePath() string {
 	return c.TokenFilePath
@@ -362,6 +374,12 @@ func (c *Configuration) getConfigOptions(logger *zap.Logger) ([]elastic.ClientOp
 	}
 	options = append(options, elastic.SetHttpClient(httpClient))
 	options = append(options, elastic.SetBasicAuth(c.Username, c.Password))
+
+	options, err := addLoggerOptions(options, c.LogLevel)
+	if err != nil {
+		return options, err
+	}
+
 	transport, err := GetHTTPRoundTripper(c, logger)
 	if err != nil {
 		return nil, err
@@ -370,6 +388,35 @@ func (c *Configuration) getConfigOptions(logger *zap.Logger) ([]elastic.ClientOp
 	return options, nil
 }
 
+func addLoggerOptions(options []elastic.ClientOptionFunc, logLevel string) ([]elastic.ClientOptionFunc, error) {
+	// Decouple ES logger from the log-level assigned to the parent application's log-level; otherwise, the least
+	// permissive log-level will dominate.
+	// e.g. --log-level=info and --es.log-level=debug would mute ES's debug logging and would require --log-level=debug
+	// to show ES debug logs.
+	prodConfig := zap.NewProductionConfig()
+	prodConfig.Level.SetLevel(zap.DebugLevel)
+
+	esLogger, err := prodConfig.Build()
+	if err != nil {
+		return options, err
+	}
+
+	// Elastic client requires a "Printf"-able logger.
+	l := zapgrpc.NewLogger(esLogger)
+	switch logLevel {
+	case "debug":
+		l = zapgrpc.NewLogger(esLogger, zapgrpc.WithDebug())
+		options = append(options, elastic.SetTraceLog(l))
+	case "info":
+		options = append(options, elastic.SetInfoLog(l))
+	case "error":
+		options = append(options, elastic.SetErrorLog(l))
+	default:
+		return options, fmt.Errorf("unrecognized log-level: \"%s\"", logLevel)
+	}
+	return options, nil
+}
+
 // GetHTTPRoundTripper returns configured http.RoundTripper
 func GetHTTPRoundTripper(c *Configuration, logger *zap.Logger) (http.RoundTripper, error) {
 	if c.TLS.Enabled {

plugin/storage/es/options.go

@@ -56,6 +56,7 @@ const (
 	suffixEnabled             = ".enabled"
 	suffixVersion             = ".version"
 	suffixMaxDocCount         = ".max-doc-count"
+	suffixLogLevel            = ".log-level"
 	// default number of documents to return from a query (elasticsearch allowed limit)
 	// see search.max_buckets and index.max_result_window
 	defaultMaxDocCount = 10_000
@@ -102,6 +103,7 @@ func NewOptions(primaryNamespace string, otherNamespaces ...string) *Options {
 		Version:              0,
 		Servers:              []string{defaultServerURL},
 		MaxDocCount:          defaultMaxDocCount,
+		LogLevel:             "error",
 	}
 	options := &Options{
 		Primary: namespaceConfig{
@@ -240,6 +242,10 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) {
 		nsConfig.namespace+suffixMaxDocCount,
 		nsConfig.MaxDocCount,
 		"The maximum document count to return from an Elasticsearch query. This will also apply to aggregations.")
+	flagSet.String(
+		nsConfig.namespace+suffixLogLevel,
+		nsConfig.LogLevel,
+		"The Elasticsearch client log-level. Valid levels: [debug, info, error]")
 
 	if nsConfig.namespace == archiveNamespace {
 		flagSet.Bool(
@@ -290,6 +296,7 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) {
 	cfg.Enabled = v.GetBool(cfg.namespace + suffixEnabled)
 	cfg.CreateIndexTemplates = v.GetBool(cfg.namespace + suffixCreateIndexTemplate)
 	cfg.Version = uint(v.GetInt(cfg.namespace + suffixVersion))
+	cfg.LogLevel = v.GetString(cfg.namespace + suffixLogLevel)
 
 	cfg.MaxDocCount = v.GetInt(cfg.namespace + suffixMaxDocCount)
 	cfg.UseILM = v.GetBool(cfg.namespace + suffixUseILM)