Add basic wildcard/keyword fallback for upcoming ECS releases

CHANGELOG.next.asciidoc

@@ -524,6 +524,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add proxy metricset for istio module. {pull}21751[21751]
 - Added Kafka version 2.2 to the list of supported versions. {pull}22328[22328]
 - Add support for ephemeral containers in kubernetes autodiscover and `add_kubernetes_metadata`. {pull}22389[22389] {pull}22439[22439]
+- Added support for wildcard fields and keyword fallback in beats setup commands. {pull}22521[22521]
 
 *Auditbeat*
 

auditbeat/cmd/root.go

@@ -54,19 +54,29 @@ var withECSVersion = processing.WithFields(common.MapStr{
 	},
 })
 
-func init() {
-	create := beater.Creator(
-		beater.WithModuleOptions(
-			module.WithEventModifier(core.AddDatasetToEvent),
-		),
-	)
+// AuditbeatSettings contains the default settings for auditbeat
+func AuditbeatSettings() instance.Settings {
 	var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError)
-	settings := instance.Settings{
+	return instance.Settings{
 		RunFlags:      runFlags,
 		Name:          Name,
 		HasDashboards: true,
 		Processing:    processing.MakeDefaultSupport(true, withECSVersion, processing.WithHost, processing.WithAgentMeta()),
 	}
-	RootCmd = cmd.GenRootCmdWithSettings(create, settings)
-	RootCmd.AddCommand(ShowCmd)
+}
+
+// Initialize initializes the entrypoint commands for journalbeat
+func Initialize(settings instance.Settings) *cmd.BeatsRootCmd {
+	create := beater.Creator(
+		beater.WithModuleOptions(
+			module.WithEventModifier(core.AddDatasetToEvent),
+		),
+	)
+	rootCmd := cmd.GenRootCmdWithSettings(create, settings)
+	rootCmd.AddCommand(ShowCmd)
+	return rootCmd
+}
+
+func init() {
+	RootCmd = Initialize(AuditbeatSettings())
 }

auditbeat/main_test.go

@@ -45,5 +45,5 @@ func TestSystem(t *testing.T) {
 }
 
 func TestTemplate(t *testing.T) {
-	template.TestTemplate(t, cmd.Name)
+	template.TestTemplate(t, cmd.Name, false)
 }

filebeat/cmd/root.go

@@ -35,17 +35,23 @@ import (
 // Name of this beat
 const Name = "filebeat"
 
-// Filebeat build the beat root command for executing filebeat and it's subcommands.
-func Filebeat(inputs beater.PluginFactory) *cmd.BeatsRootCmd {
+// RootCmd to handle beats cli
+var RootCmd *cmd.BeatsRootCmd
+
+// FilebeatSettings contains the default settings for filebeat
+func FilebeatSettings() instance.Settings {
 	var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError)
 	runFlags.AddGoFlag(flag.CommandLine.Lookup("once"))
 	runFlags.AddGoFlag(flag.CommandLine.Lookup("modules"))
-	settings := instance.Settings{
+	return instance.Settings{
 		RunFlags:      runFlags,
 		Name:          Name,
 		HasDashboards: true,
 	}
+}
 
+// Filebeat build the beat root command for executing filebeat and it's subcommands.
+func Filebeat(inputs beater.PluginFactory, settings instance.Settings) *cmd.BeatsRootCmd {
 	command := cmd.GenRootCmdWithSettings(beater.New(inputs), settings)
 	command.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("M"))
 	command.TestCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))

filebeat/main.go

@@ -33,7 +33,7 @@ import (
 // Finally, input uses the registrar information, on restart, to
 // determine where in each file to restart a harvester.
 func main() {
-	if err := cmd.Filebeat(inputs.Init).Execute(); err != nil {
+	if err := cmd.Filebeat(inputs.Init, cmd.FilebeatSettings()).Execute(); err != nil {
 		os.Exit(1)
 	}
 }

filebeat/main_test.go

@@ -36,7 +36,7 @@ var fbCommand *cmd.BeatsRootCmd
 func init() {
 	testing.Init()
 	systemTest = flag.Bool("systemTest", false, "Set to true when running system tests")
-	fbCommand = fbcmd.Filebeat(inputs.Init)
+	fbCommand = fbcmd.Filebeat(inputs.Init, fbcmd.FilebeatSettings())
 	fbCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest"))
 	fbCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile"))
 }
@@ -51,5 +51,5 @@ func TestSystem(t *testing.T) {
 }
 
 func TestTemplate(t *testing.T) {
-	template.TestTemplate(t, fbCommand.Name())
+	template.TestTemplate(t, fbCommand.Name(), false)
 }

heartbeat/cmd/root.go

@@ -38,23 +38,28 @@ var Name = "heartbeat"
 // RootCmd to handle beats cli
 var RootCmd *cmd.BeatsRootCmd
 
-func init() {
-	settings := instance.Settings{
+// HeartbeatSettings contains the default settings for heartbeat
+func HeartbeatSettings() instance.Settings {
+	return instance.Settings{
 		Name:          Name,
 		Processing:    processing.MakeDefaultSupport(true, processing.WithECS, processing.WithAgentMeta()),
 		HasDashboards: false,
 	}
-	RootCmd = cmd.GenRootCmdWithSettings(beater.New, settings)
+}
+
+// Initialize initializes the entrypoint commands for heartbeat
+func Initialize(settings instance.Settings) *cmd.BeatsRootCmd {
+	rootCmd := cmd.GenRootCmdWithSettings(beater.New, settings)
 
 	// remove dashboard from export commands
-	for _, cmd := range RootCmd.ExportCmd.Commands() {
+	for _, cmd := range rootCmd.ExportCmd.Commands() {
 		if cmd.Name() == "dashboard" {
-			RootCmd.ExportCmd.RemoveCommand(cmd)
+			rootCmd.ExportCmd.RemoveCommand(cmd)
 		}
 	}
 
 	// only add defined flags to setup command
-	setup := RootCmd.SetupCmd
+	setup := rootCmd.SetupCmd
 	setup.Short = "Setup Elasticsearch index template and pipelines"
 	setup.Long = `This command does initial setup of the environment:
  * Index mapping template in Elasticsearch to ensure fields are mapped.
@@ -66,4 +71,10 @@ func init() {
 	setup.Flags().MarkDeprecated(cmd.ILMPolicyKey, fmt.Sprintf("use --%s instead", cmd.IndexManagementKey))
 	setup.Flags().Bool(cmd.TemplateKey, false, "Setup index template")
 	setup.Flags().Bool(cmd.ILMPolicyKey, false, "Setup ILM policy")
+
+	return rootCmd
+}
+
+func init() {
+	RootCmd = Initialize(HeartbeatSettings())
 }

heartbeat/main_test.go

@@ -44,5 +44,5 @@ func TestSystem(t *testing.T) {
 }
 
 func TestTemplate(t *testing.T) {
-	template.TestTemplate(t, cmd.Name)
+	template.TestTemplate(t, cmd.Name, false)
 }

journalbeat/cmd/root.go

@@ -32,4 +32,18 @@ import (
 var Name = "journalbeat"
 
 // RootCmd to handle beats cli
-var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{Name: Name, HasDashboards: false})
+var RootCmd *cmd.BeatsRootCmd
+
+// JournalbeatSettings contains the default settings for journalbeat
+func JournalbeatSettings() instance.Settings {
+	return instance.Settings{Name: Name, HasDashboards: false}
+}
+
+// Initialize initializes the entrypoint commands for journalbeat
+func Initialize(settings instance.Settings) *cmd.BeatsRootCmd {
+	return cmd.GenRootCmdWithSettings(beater.New, settings)
+}
+
+func init() {
+	RootCmd = Initialize(JournalbeatSettings())
+}

journalbeat/main_test.go

@@ -45,5 +45,5 @@ func TestSystem(t *testing.T) {
 }
 
 func TestTemplate(t *testing.T) {
-	template.TestTemplate(t, cmd.Name)
+	template.TestTemplate(t, cmd.Name, false)
 }

libbeat/beat/info.go

@@ -21,13 +21,14 @@ import "github.com/gofrs/uuid"
 
 // Info stores a beats instance meta data.
 type Info struct {
-	Beat        string    // The actual beat's name
-	IndexPrefix string    // The beat's index prefix in Elasticsearch.
-	Version     string    // The beat version. Defaults to the libbeat version when an implementation does not set a version
-	Name        string    // configured beat name
-	Hostname    string    // hostname
-	ID          uuid.UUID // ID assigned to beat machine
-	EphemeralID uuid.UUID // ID assigned to beat process invocation (PID)
+	Beat            string    // The actual beat's name
+	IndexPrefix     string    // The beat's index prefix in Elasticsearch.
+	Version         string    // The beat version. Defaults to the libbeat version when an implementation does not set a version
+	ElasticLicensed bool      // Whether the beat is licensed under and Elastic License
+	Name            string    // configured beat name
+	Hostname        string    // hostname
+	ID              uuid.UUID // ID assigned to beat machine
+	EphemeralID     uuid.UUID // ID assigned to beat process invocation (PID)
 
 	// Monitoring-related fields
 	Monitoring struct {

libbeat/cmd/instance/beat.go

@@ -156,6 +156,7 @@ func Run(settings Settings, bt beat.Creator) error {
 	name := settings.Name
 	idxPrefix := settings.IndexPrefix
 	version := settings.Version
+	elasticLicensed := settings.ElasticLicensed
 
 	return handleError(func() error {
 		defer func() {
@@ -164,7 +165,7 @@ func Run(settings Settings, bt beat.Creator) error {
 					"panic", r, zap.Stack("stack"))
 			}
 		}()
-		b, err := NewBeat(name, idxPrefix, version)
+		b, err := NewBeat(name, idxPrefix, version, elasticLicensed)
 		if err != nil {
 			return err
 		}
@@ -191,7 +192,7 @@ func Run(settings Settings, bt beat.Creator) error {
 
 // NewInitializedBeat creates a new beat where all information and initialization is derived from settings
 func NewInitializedBeat(settings Settings) (*Beat, error) {
-	b, err := NewBeat(settings.Name, settings.IndexPrefix, settings.Version)
+	b, err := NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed)
 	if err != nil {
 		return nil, err
 	}
@@ -202,7 +203,7 @@ func NewInitializedBeat(settings Settings) (*Beat, error) {
 }
 
 // NewBeat creates a new beat instance
-func NewBeat(name, indexPrefix, v string) (*Beat, error) {
+func NewBeat(name, indexPrefix, v string, elasticLicensed bool) (*Beat, error) {
 	if v == "" {
 		v = version.GetDefaultVersion()
 	}
@@ -227,13 +228,14 @@ func NewBeat(name, indexPrefix, v string) (*Beat, error) {
 
 	b := beat.Beat{
 		Info: beat.Info{
-			Beat:        name,
-			IndexPrefix: indexPrefix,
-			Version:     v,
-			Name:        hostname,
-			Hostname:    hostname,
-			ID:          id,
-			EphemeralID: ephemeralID,
+			Beat:            name,
+			ElasticLicensed: elasticLicensed,
+			IndexPrefix:     indexPrefix,
+			Version:         v,
+			Name:            hostname,
+			Hostname:        hostname,
+			ID:              id,
+			EphemeralID:     ephemeralID,
 		},
 		Fields: fields,
 	}

libbeat/cmd/instance/beat_test.go

@@ -31,7 +31,7 @@ import (
 )
 
 func TestNewInstance(t *testing.T) {
-	b, err := NewBeat("testbeat", "testidx", "0.9")
+	b, err := NewBeat("testbeat", "testidx", "0.9", false)
 	if err != nil {
 		panic(err)
 	}
@@ -45,7 +45,7 @@ func TestNewInstance(t *testing.T) {
 	assert.Equal(t, 36, len(b.Info.ID.String()))
 
 	// indexPrefix set to name if empty
-	b, err = NewBeat("testbeat", "", "0.9")
+	b, err = NewBeat("testbeat", "", "0.9", false)
 	if err != nil {
 		panic(err)
 	}
@@ -55,7 +55,7 @@ func TestNewInstance(t *testing.T) {
 }
 
 func TestNewInstanceUUID(t *testing.T) {
-	b, err := NewBeat("testbeat", "", "0.9")
+	b, err := NewBeat("testbeat", "", "0.9", false)
 	if err != nil {
 		panic(err)
 	}
@@ -69,7 +69,7 @@ func TestNewInstanceUUID(t *testing.T) {
 }
 
 func TestInitKibanaConfig(t *testing.T) {
-	b, err := NewBeat("filebeat", "testidx", "0.9")
+	b, err := NewBeat("filebeat", "testidx", "0.9", false)
 	if err != nil {
 		panic(err)
 	}
@@ -96,7 +96,7 @@ func TestInitKibanaConfig(t *testing.T) {
 }
 
 func TestEmptyMetaJson(t *testing.T) {
-	b, err := NewBeat("filebeat", "testidx", "0.9")
+	b, err := NewBeat("filebeat", "testidx", "0.9", false)
 	if err != nil {
 		panic(err)
 	}

libbeat/cmd/instance/settings.go

@@ -33,6 +33,7 @@ type Settings struct {
 	IndexPrefix     string
 	Version         string
 	HasDashboards   bool
+	ElasticLicensed bool
 	Monitoring      report.Settings
 	RunFlags        *pflag.FlagSet
 	ConfigOverrides []cfgfile.ConditionalOverride

libbeat/cmd/setup.go

@@ -58,7 +58,7 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
  * ILM policy (for Elasticsearch 6.5 and newer).
 `,
 		Run: func(cmd *cobra.Command, args []string) {
-			beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version)
+			beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err)
 				os.Exit(1)

libbeat/cmd/test/config.go

@@ -32,7 +32,7 @@ func GenTestConfigCmd(settings instance.Settings, beatCreator beat.Creator) *cob
 		Use:   "config",
 		Short: "Test configuration settings",
 		Run: func(cmd *cobra.Command, args []string) {
-			b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version)
+			b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err)
 				os.Exit(1)

libbeat/cmd/version.go

@@ -35,7 +35,7 @@ func GenVersionCmd(settings instance.Settings) *cobra.Command {
 		Short: "Show current version info",
 		Run: cli.RunWith(
 			func(_ *cobra.Command, args []string) error {
-				beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version)
+				beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed)
 				if err != nil {
 					return fmt.Errorf("error initializing beat: %s", err)
 				}

libbeat/libbeat_test.go

@@ -42,5 +42,5 @@ func TestSystem(t *testing.T) {
 }
 
 func TestTemplate(t *testing.T) {
-	template.TestTemplate(t, "mockbeat")
+	template.TestTemplate(t, "mockbeat", false)
 }

libbeat/template/load.go

@@ -183,7 +183,7 @@ func (b *templateBuilder) template(config TemplateConfig, info beat.Info, esVers
 		b.log.Info("template config not enabled")
 		return nil, nil
 	}
-	tmpl, err := New(info.Version, info.IndexPrefix, esVersion, config, migration)
+	tmpl, err := New(info.Version, info.IndexPrefix, info.ElasticLicensed, esVersion, config, migration)
 	if err != nil {
 		return nil, fmt.Errorf("error creating template instance: %v", err)
 	}