Add k8s keystore backend (elastic#18096)

(cherry picked from commit b772f2a)

CHANGELOG.next.asciidoc

@@ -295,6 +295,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `urldecode` processor to for decoding URL-encoded fields. {pull}17505[17505]
 - Add support for AWS IAM `role_arn` in credentials config. {pull}17658[17658] {issue}12464[12464]
 - Add Kerberos support to Elasticsearch output. {pull}17927[17927]
+- Add k8s keystore backend. {pull}18096[18096]
+- Add support for fixed length extraction in `dissect` processor. {pull}17191[17191]
 - Set `agent.name` to the hostname by default. {issue}16377[16377] {pull}18000[18000]
 - Add keystore support for autodiscover static configurations. {pull]16306[16306]
 - Add support for basic ECS logging. {pull}17974[17974]

deploy/kubernetes/metricbeat-kubernetes.yaml

@@ -335,6 +335,7 @@ rules:
   - namespaces
   - events
   - pods
+  - secrets
   verbs: ["get", "list", "watch"]
 - apiGroups: ["extensions"]
   resources:

deploy/kubernetes/metricbeat/metricbeat-role.yaml

@@ -11,6 +11,7 @@ rules:
   - namespaces
   - events
   - pods
+  - secrets
   verbs: ["get", "list", "watch"]
 - apiGroups: ["extensions"]
   resources:

filebeat/autodiscover/builder/hints/logs.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"regexp"
 
+	"github.com/elastic/go-ucfg"
+
 	"github.com/elastic/beats/v7/filebeat/fileset"
 	"github.com/elastic/beats/v7/filebeat/harvester"
 	"github.com/elastic/beats/v7/libbeat/autodiscover"
@@ -70,7 +72,7 @@ func NewLogHints(cfg *common.Config) (autodiscover.Builder, error) {
 }
 
 // Create config based on input hints in the bus event
-func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
+func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*common.Config {
 	var hints common.MapStr
 	hIface, ok := event["hints"]
 	if ok {
@@ -109,7 +111,7 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
 		}
 		logp.Debug("hints.builder", "generated config %+v", configs)
 		// Apply information in event to the template to generate the final config
-		return template.ApplyConfigTemplate(event, configs, false)
+		return template.ApplyConfigTemplate(event, configs)
 	}
 
 	tempCfg := common.MapStr{}
@@ -163,7 +165,7 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
 	logp.Debug("hints.builder", "generated config %+v", config)
 
 	// Apply information in event to the template to generate the final config
-	return template.ApplyConfigTemplate(event, []*common.Config{config}, false)
+	return template.ApplyConfigTemplate(event, []*common.Config{config})
 }
 
 func (l *logHints) getMultiline(hints common.MapStr) common.MapStr {

heartbeat/autodiscover/builder/hints/monitors.go

@@ -23,6 +23,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/elastic/go-ucfg"
+
 	"github.com/elastic/beats/v7/libbeat/autodiscover"
 	"github.com/elastic/beats/v7/libbeat/autodiscover/builder"
 	"github.com/elastic/beats/v7/libbeat/autodiscover/template"
@@ -60,7 +62,7 @@ func NewHeartbeatHints(cfg *common.Config) (autodiscover.Builder, error) {
 }
 
 // Create config based on input hints in the bus event
-func (hb *heartbeatHints) CreateConfig(event bus.Event) []*common.Config {
+func (hb *heartbeatHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*common.Config {
 	var hints common.MapStr
 	hIface, ok := event["hints"]
 	if ok {
@@ -91,7 +93,7 @@ func (hb *heartbeatHints) CreateConfig(event bus.Event) []*common.Config {
 		}
 		hb.logger.Debugf("generated config %+v", configs)
 		// Apply information in event to the template to generate the final config
-		return template.ApplyConfigTemplate(event, configs, false)
+		return template.ApplyConfigTemplate(event, configs)
 	}
 
 	tempCfg := common.MapStr{}
@@ -121,7 +123,7 @@ func (hb *heartbeatHints) CreateConfig(event bus.Event) []*common.Config {
 	}
 
 	// Apply information in event to the template to generate the final config
-	return template.ApplyConfigTemplate(event, configs, false)
+	return template.ApplyConfigTemplate(event, configs)
 }
 
 func (hb *heartbeatHints) getType(hints common.MapStr) common.MapStr {

libbeat/autodiscover/appenders/config/config.go

@@ -104,7 +104,7 @@ func (c *configAppender) Append(event bus.Event) {
 		}
 
 		// Apply the template
-		template.ApplyConfigTemplate(event, cfgs, false)
+		template.ApplyConfigTemplate(event, cfgs)
 	}
 
 	// Replace old config with newly appended configs

libbeat/autodiscover/autodiscover.go

@@ -191,7 +191,6 @@ func (a *Autodiscover) handleStart(event bus.Event) bool {
 	}
 
 	if a.logger.IsDebug() {
-
 		for _, c := range configs {
 			a.logger.Debugf("Generated config: %+v", common.DebugString(c, true))
 		}

libbeat/autodiscover/builder.go

@@ -22,18 +22,25 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/elastic/go-ucfg"
+
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/bus"
+	"github.com/elastic/beats/v7/libbeat/keystore"
 )
 
 // Builder provides an interface by which configs can be built from provider metadata
 type Builder interface {
 	// CreateConfig creates a config from hints passed from providers
-	CreateConfig(event bus.Event) []*common.Config
+	CreateConfig(event bus.Event, options ...ucfg.Option) []*common.Config
 }
 
-// Builders is a list of Builder objects
-type Builders []Builder
+// builders is a struct of Builder list objects and a `keystoreProvider`, which
+// has access to a keystores registry
+type Builders struct {
+	builders         []Builder
+	keystoreProvider keystore.Provider
+}
 
 // BuilderConstructor is a func used to generate a Builder object
 type BuilderConstructor func(*common.Config) (Builder, error)
@@ -89,9 +96,18 @@ func (r *registry) BuildBuilder(c *common.Config) (Builder, error) {
 // GetConfig creates configs for all builders initialized.
 func (b Builders) GetConfig(event bus.Event) []*common.Config {
 	configs := []*common.Config{}
-
-	for _, builder := range b {
-		if config := builder.CreateConfig(event); config != nil {
+	var opts []ucfg.Option
+
+	if b.keystoreProvider != nil {
+		k8sKeystore := b.keystoreProvider.GetKeystore(event)
+		if k8sKeystore != nil {
+			opts = []ucfg.Option{
+				ucfg.Resolve(keystore.ResolverWrap(k8sKeystore)),
+			}
+		}
+	}
+	for _, builder := range b.builders {
+		if config := builder.CreateConfig(event, opts...); config != nil {
 			configs = append(configs, config...)
 		}
 	}
@@ -100,12 +116,16 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config {
 }
 
 // NewBuilders instances the given list of builders. hintsCfg holds `hints` settings
-// for simplified mode (single 'hints' builder)
-func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config) (Builders, error) {
+// for simplified mode (single 'hints' builder), `keystoreProvider` has access to keystore registry
+func NewBuilders(
+	bConfigs []*common.Config,
+	hintsCfg *common.Config,
+	keystoreProvider keystore.Provider,
+) (Builders, error) {
 	var builders Builders
 	if hintsCfg.Enabled() {
 		if len(bConfigs) > 0 {
-			return nil, errors.New("hints.enabled is incompatible with manually defining builders")
+			return Builders{}, errors.New("hints.enabled is incompatible with manually defining builders")
 		}
 
 		// pass rest of hints settings to the builder
@@ -116,10 +136,10 @@ func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config) (Builders,
 	for _, bcfg := range bConfigs {
 		builder, err := Registry.BuildBuilder(bcfg)
 		if err != nil {
-			return nil, err
+			return Builders{}, err
 		}
-		builders = append(builders, builder)
+		builders.builders = append(builders.builders, builder)
 	}
-
+	builders.keystoreProvider = keystoreProvider
 	return builders, nil
 }

libbeat/autodiscover/builder_test.go

@@ -22,13 +22,15 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/elastic/go-ucfg"
+
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/bus"
 )
 
 type fakeBuilder struct{}
 
-func (f *fakeBuilder) CreateConfig(event bus.Event) []*common.Config {
+func (f *fakeBuilder) CreateConfig(event bus.Event, options ...ucfg.Option) []*common.Config {
 	return []*common.Config{common.NewConfig()}
 }
 
@@ -65,7 +67,7 @@ func TestBuilderRegistry(t *testing.T) {
 	assert.Equal(t, len(res), 1)
 
 	builders := Builders{}
-	builders = append(builders, builder)
+	builders.builders = append(builders.builders, builder)
 
 	// Try using builders object for the same as above and expect
 	// the same result

libbeat/autodiscover/providers/docker/docker.go

@@ -56,7 +56,6 @@ type Provider struct {
 	stoppers      map[string]*time.Timer
 	stopTrigger   chan *dockerContainerMetadata
 	logger        *logp.Logger
-	keystore      keystore.Keystore
 }
 
 // AutodiscoverBuilder builds and returns an autodiscover provider
@@ -78,15 +77,15 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore
 		return nil, errWrap(err)
 	}
 
-	mapper, err := template.NewConfigMapper(config.Templates)
+	mapper, err := template.NewConfigMapper(config.Templates, keystore, nil)
 	if err != nil {
 		return nil, errWrap(err)
 	}
-	if len(mapper) == 0 && !config.Hints.Enabled() {
+	if len(mapper.ConditionMaps) == 0 && !config.Hints.Enabled() {
 		return nil, errWrap(fmt.Errorf("no configs or hints defined for autodiscover provider"))
 	}
 
-	builders, err := autodiscover.NewBuilders(config.Builders, config.Hints)
+	builders, err := autodiscover.NewBuilders(config.Builders, config.Hints, nil)
 	if err != nil {
 		return nil, errWrap(err)
 	}
@@ -117,7 +116,6 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore
 		stoppers:      make(map[string]*time.Timer),
 		stopTrigger:   make(chan *dockerContainerMetadata),
 		logger:        logger,
-		keystore:      keystore,
 	}, nil
 }
 
@@ -306,8 +304,6 @@ func (d *Provider) emitContainer(container *docker.Container, meta *dockerMetada
 }
 
 func (d *Provider) publish(event bus.Event) {
-	// attach keystore to the event to be consumed by the static configs
-	event["keystore"] = d.keystore
 	// Try to match a config
 	if config := d.templates.GetConfig(event); config != nil {
 		event["config"] = config

libbeat/autodiscover/providers/jolokia/jolokia.go

@@ -49,7 +49,6 @@ type Provider struct {
 	appenders autodiscover.Appenders
 	templates template.Mapper
 	discovery DiscoveryProber
-	keystore  keystore.Keystore
 }
 
 // AutodiscoverBuilder builds a Jolokia Discovery autodiscover provider, it fails if
@@ -70,15 +69,15 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore
 		Interfaces:   config.Interfaces,
 	}
 
-	mapper, err := template.NewConfigMapper(config.Templates)
+	mapper, err := template.NewConfigMapper(config.Templates, keystore, nil)
 	if err != nil {
 		return nil, errWrap(err)
 	}
-	if len(mapper) == 0 {
+	if len(mapper.ConditionMaps) == 0 {
 		return nil, errWrap(fmt.Errorf("no configs defined for autodiscover provider"))
 	}
 
-	builders, err := autodiscover.NewBuilders(config.Builders, nil)
+	builders, err := autodiscover.NewBuilders(config.Builders, nil, nil)
 	if err != nil {
 		return nil, errWrap(err)
 	}
@@ -94,7 +93,6 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore
 		builders:  builders,
 		appenders: appenders,
 		discovery: discovery,
-		keystore:  keystore,
 	}, nil
 }
 
@@ -109,8 +107,6 @@ func (p *Provider) Start() {
 }
 
 func (p *Provider) publish(event bus.Event) {
-	// attach keystore to the event to be consumed by the static configs
-	event["keystore"] = p.keystore
 	if config := p.templates.GetConfig(event); config != nil {
 		event["config"] = config
 	} else if config := p.builders.GetConfig(event); config != nil {

libbeat/autodiscover/providers/kubernetes/config_test.go

@@ -22,6 +22,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/elastic/go-ucfg"
+
 	"github.com/elastic/beats/v7/libbeat/autodiscover"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/bus"
@@ -76,6 +78,6 @@ func newMockBuilder(_ *common.Config) (autodiscover.Builder, error) {
 	return &mockBuilder{}, nil
 }
 
-func (m *mockBuilder) CreateConfig(event bus.Event) []*common.Config {
+func (m *mockBuilder) CreateConfig(event bus.Event, options ...ucfg.Option) []*common.Config {
 	return nil
 }

libbeat/autodiscover/providers/kubernetes/kubernetes.go

@@ -30,6 +30,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/bus"
 	"github.com/elastic/beats/v7/libbeat/common/kubernetes"
+	"github.com/elastic/beats/v7/libbeat/common/kubernetes/k8skeystore"
 	"github.com/elastic/beats/v7/libbeat/keystore"
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
@@ -55,7 +56,6 @@ type Provider struct {
 	appenders autodiscover.Appenders
 	logger    *logp.Logger
 	eventer   Eventer
-	keystore  keystore.Keystore
 }
 
 // AutodiscoverBuilder builds and returns an autodiscover provider
@@ -77,12 +77,14 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore
 		return nil, errWrap(err)
 	}
 
-	mapper, err := template.NewConfigMapper(config.Templates)
+	k8sKeystoreProvider := k8skeystore.NewKubernetesKeystoresRegistry(logger, client)
+
+	mapper, err := template.NewConfigMapper(config.Templates, keystore, k8sKeystoreProvider)
 	if err != nil {
 		return nil, errWrap(err)
 	}
 
-	builders, err := autodiscover.NewBuilders(config.Builders, config.Hints)
+	builders, err := autodiscover.NewBuilders(config.Builders, config.Hints, k8sKeystoreProvider)
 	if err != nil {
 		return nil, errWrap(err)
 	}
@@ -99,7 +101,6 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore
 		builders:  builders,
 		appenders: appenders,
 		logger:    logger,
-		keystore:  keystore,
 	}
 
 	switch config.Resource {
@@ -138,8 +139,6 @@ func (p *Provider) String() string {
 }
 
 func (p *Provider) publish(event bus.Event) {
-	// attach keystore to the event to be consumed by the static configs
-	event["keystore"] = p.keystore
 	// Try to match a config
 	if config := p.templates.GetConfig(event); config != nil {
 		event["config"] = config