From 3dfb1f74d8a9f36e771573190457d394b5f381aa Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 28 Apr 2020 14:16:32 +0200 Subject: [PATCH 1/6] introspection --- .../pkg/agent/application/emitter.go | 6 +- .../application/introspect_config_cmd.go | 129 +++++++++++ .../application/introspect_output_cmd.go | 210 ++++++++++++++++++ x-pack/elastic-agent/pkg/agent/cmd/common.go | 1 + .../elastic-agent/pkg/agent/cmd/introspect.go | 69 ++++++ 5 files changed, 414 insertions(+), 1 deletion(-) create mode 100644 x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go create mode 100644 x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go create mode 100644 x-pack/elastic-agent/pkg/agent/cmd/introspect.go diff --git a/x-pack/elastic-agent/pkg/agent/application/emitter.go b/x-pack/elastic-agent/pkg/agent/application/emitter.go index 2279d78565d9..249acdd213fb 100644 --- a/x-pack/elastic-agent/pkg/agent/application/emitter.go +++ b/x-pack/elastic-agent/pkg/agent/application/emitter.go @@ -26,7 +26,11 @@ type configModifiers struct { Decorators []decoratorFunc } -func emitter(log *logger.Logger, router *router, modifiers *configModifiers, reloadables ...reloadable) emitterFunc { +type programsDispatcher interface { + Dispatch(id string, grpProg map[routingKey][]program.Program) error +} + +func emitter(log *logger.Logger, router programsDispatcher, modifiers *configModifiers, reloadables ...reloadable) emitterFunc { return func(c *config.Config) error { if err := InjectAgentConfig(c); err != nil { return err diff --git a/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go b/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go new file mode 100644 index 000000000000..e63d675a9ef8 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go @@ -0,0 +1,129 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package application + +import ( + "fmt" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" + yaml "gopkg.in/yaml.v2" +) + +// IntrospectConfigCmd is an introspect subcommand that shows configurations of the agent. +type IntrospectConfigCmd struct { + cfgPath string +} + +// NewIntrospectConfigCmd creates a new introspect command. +func NewIntrospectConfigCmd(configPath string, +) (*IntrospectConfigCmd, error) { + return &IntrospectConfigCmd{ + cfgPath: configPath, + }, nil +} + +// Execute tries to enroll the agent into Fleet. +func (c *IntrospectConfigCmd) Execute() error { + return c.introspectConfig() +} + +func (c *IntrospectConfigCmd) introspectConfig() error { + cfg, err := loadConfig(c.cfgPath) + if err != nil { + return err + } + + isLocal, err := isLocalMode(cfg) + if err != nil { + return err + } + + if isLocal { + return printConfig(cfg) + } + + fleetConfig, err := loadFleetConfig(cfg) + if err != nil { + return err + } else if fleetConfig == nil { + return errors.New("no fleet config retrieved yet") + } + + return printMapStringConfig(fleetConfig) +} + +func loadConfig(configPath string) (*config.Config, error) { + rawConfig, err := config.LoadYAML(configPath) + if err != nil { + return nil, err + } + + if err := InjectAgentConfig(rawConfig); err != nil { + return nil, err + } + + return rawConfig, nil +} + +func loadFleetConfig(cfg *config.Config) (map[string]interface{}, error) { + log, err := newErrorLogger() + if err != nil { + return nil, err + } + + as, err := newActionStore(log, storage.NewDiskStore(fleetActionStoreFile())) + if err != nil { + return nil, err + } + + for _, c := range as.Actions() { + cfgChange, ok := c.(*fleetapi.ActionConfigChange) + if !ok { + continue + } + + fmt.Println("Action ID:", cfgChange.ID()) + return cfgChange.Config, nil + } + return nil, nil +} + +func isLocalMode(rawConfig *config.Config) (bool, error) { + c := localDefaultConfig() + if err := rawConfig.Unpack(&c); err != nil { + return false, errors.New(err, "initiating application") + } + + managementConfig := struct { + Mode string `config:"mode" yaml:"mode"` + }{} + + if err := c.Management.Unpack(&managementConfig); err != nil { + return false, errors.New(err, "initiating application") + } + return managementConfig.Mode == "local", nil +} + +func printMapStringConfig(mapStr map[string]interface{}) error { + data, err := yaml.Marshal(mapStr) + if err != nil { + return errors.New(err, "could not marshal to YAML") + } + + fmt.Println(string(data)) + return nil +} + +func printConfig(cfg *config.Config) error { + mapStr, err := cfg.ToMapStr() + if err != nil { + return err + } + + return printMapStringConfig(mapStr) +} diff --git a/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go b/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go new file mode 100644 index 000000000000..4a4966b21414 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go @@ -0,0 +1,210 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package application + +import ( + "fmt" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/filters" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/plugin/app/monitoring/noop" + "github.com/urso/ecslog" + "github.com/urso/ecslog/backend" + "github.com/urso/ecslog/backend/appender" + "github.com/urso/ecslog/backend/layout" +) + +// IntrospectOutputCmd is an introspect subcommand that shows configurations of the agent. +type IntrospectOutputCmd struct { + cfgPath string + output string + program string +} + +// NewIntrospectOutputCmd creates a new introspect command. +func NewIntrospectOutputCmd(configPath, output, program string) (*IntrospectOutputCmd, error) { + return &IntrospectOutputCmd{ + cfgPath: configPath, + output: output, + program: program, + }, nil +} + +// Execute tries to enroll the agent into Fleet. +func (c *IntrospectOutputCmd) Execute() error { + if c.output == "" { + return c.introspectOutputs() + } + + return c.introspectOutput() +} + +func (c *IntrospectOutputCmd) introspectOutputs() error { + cfg, err := loadConfig(c.cfgPath) + if err != nil { + return err + } + + isLocal, err := isLocalMode(cfg) + if err != nil { + return err + } + + l, err := newErrorLogger() + if err != nil { + return err + } + + if isLocal { + return listOutputsFromConfig(l, cfg) + } + + fleetConfig, err := loadFleetConfig(cfg) + if err != nil { + return err + } else if fleetConfig == nil { + return errors.New("no fleet config retrieved yet") + } + + return listOutputsFromMap(l, fleetConfig) +} + +func listOutputsFromConfig(log *logger.Logger, cfg *config.Config) error { + programsGroup, err := getProgramsFromConfig(log, cfg) + if err != nil { + return err + + } + + for k := range programsGroup { + fmt.Println(k) + } + + return nil +} + +func listOutputsFromMap(log *logger.Logger, cfg map[string]interface{}) error { + c, err := config.NewConfigFrom(cfg) + if err != nil { + return err + } + + return listOutputsFromConfig(log, c) +} + +func (c *IntrospectOutputCmd) introspectOutput() error { + cfg, err := loadConfig(c.cfgPath) + if err != nil { + return err + } + + l, err := newErrorLogger() + if err != nil { + return err + } + + isLocal, err := isLocalMode(cfg) + if err != nil { + return err + } + + if isLocal { + return printOutputFromConfig(l, c.output, c.program, cfg) + } + + fleetConfig, err := loadFleetConfig(cfg) + if err != nil { + return err + } else if fleetConfig == nil { + return errors.New("no fleet config retrieved yet") + } + + return printOutputFromMap(l, c.output, c.program, fleetConfig) +} + +func printOutputFromConfig(log *logger.Logger, output, programName string, cfg *config.Config) error { + programsGroup, err := getProgramsFromConfig(log, cfg) + if err != nil { + return err + + } + + for k, programs := range programsGroup { + if k != output { + continue + } + + var programFound bool + for _, p := range programs { + if programName != "" && programName != p.Spec.Cmd { + continue + } + + programFound = true + fmt.Printf("[%s] %s:\n", k, p.Spec.Cmd) + printMapStringConfig(p.Configuration()) + fmt.Println("---") + } + + if !programFound { + fmt.Printf("program '%s' is not recognized within output '%s', try running `elastic-agent introspect output` to find available outputs.\n", + programName, + output) + } + return nil + } + + fmt.Printf("output '%s' is not recognized, try running `elastic-agent introspect output` to find available outputs.\n", output) + + return nil +} + +func printOutputFromMap(log *logger.Logger, output, programName string, cfg map[string]interface{}) error { + c, err := config.NewConfigFrom(cfg) + if err != nil { + return err + } + + return printOutputFromConfig(log, output, programName, c) +} + +func getProgramsFromConfig(log *logger.Logger, cfg *config.Config) (map[string][]program.Program, error) { + monitor := noop.NewMonitor() + router := &inmemRouter{} + emit := emitter( + log, + router, + &configModifiers{ + Decorators: []decoratorFunc{injectMonitoring}, + Filters: []filterFunc{filters.ConstraintFilter}, + }, + monitor, + ) + + if err := emit(cfg); err != nil { + return nil, err + } + return router.programs, nil +} + +type inmemRouter struct { + programs map[string][]program.Program +} + +func (r *inmemRouter) Dispatch(id string, grpProg map[routingKey][]program.Program) error { + r.programs = grpProg + return nil +} + +func newErrorLogger() (*logger.Logger, error) { + backend, err := appender.Console(backend.Error, layout.Text(true)) + if err != nil { + return nil, err + } + return ecslog.New(backend), nil +} diff --git a/x-pack/elastic-agent/pkg/agent/cmd/common.go b/x-pack/elastic-agent/pkg/agent/cmd/common.go index 0189f8e408d8..af366c7f782f 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/common.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/common.go @@ -61,6 +61,7 @@ func NewCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Command { cmd.AddCommand(basecmd.NewDefaultCommandsWithArgs(args, streams)...) cmd.AddCommand(newRunCommandWithArgs(flags, args, streams)) cmd.AddCommand(newEnrollCommandWithArgs(flags, args, streams)) + cmd.AddCommand(newIntrospectCommandWithArgs(flags, args, streams)) return cmd } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/introspect.go b/x-pack/elastic-agent/pkg/agent/cmd/introspect.go new file mode 100644 index 000000000000..6bcd3418cbff --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/cmd/introspect.go @@ -0,0 +1,69 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli" +) + +func newIntrospectCommandWithArgs(flags *globalFlags, s []string, streams *cli.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "introspect", + Short: "Shows configuration of the agent", + Long: "Shows current configuration of the agent", + Args: cobra.ExactArgs(0), + Run: func(c *cobra.Command, args []string) { + command, err := application.NewIntrospectConfigCmd("elastic-agent.yml") + if err != nil { + fmt.Fprintf(streams.Err, "%v\n", err) + os.Exit(1) + } + + if err := command.Execute(); err != nil { + fmt.Fprintf(streams.Err, "%v\n", err) + os.Exit(1) + } + }, + } + + cmd.AddCommand(newIntrospectOutputCommandWithArgs(flags, s, streams)) + + return cmd +} + +func newIntrospectOutputCommandWithArgs(flags *globalFlags, _ []string, streams *cli.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "output", + Short: "Displays configuration generated for output", + Long: "Displays configuration generated for output.\nIf no output is specified list of output is displayed", + Args: cobra.MaximumNArgs(2), + Run: func(c *cobra.Command, args []string) { + outName, _ := c.Flags().GetString("output") + program, _ := c.Flags().GetString("program") + + command, err := application.NewIntrospectOutputCmd("elastic-agent.yml", outName, program) + if err != nil { + fmt.Fprintf(streams.Err, "%v\n", err) + os.Exit(1) + } + + if err := command.Execute(); err != nil { + fmt.Fprintf(streams.Err, "%v\n", err) + os.Exit(1) + } + }, + } + + cmd.Flags().StringP("output", "o", "", "name of the output to be introspected") + cmd.Flags().StringP("program", "p", "", "type of program to introspect, needs to be combined with output. e.g filebeat") + + return cmd +} From b03a49da86036d80ea0b36b8ab356e150fccd3c8 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 30 Apr 2020 10:06:00 +0200 Subject: [PATCH 2/6] changelog --- x-pack/elastic-agent/CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index bb57c61f03e2..63c2e33b1a00 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -43,3 +43,4 @@ - Enable Filebeat input: S3, Azureeventhub, cloudfoundry, httpjson, netflow, o365audit. {pull}17909[17909] - Use data subfolder as default for process logs {pull}17960[17960] - Enable debug log level for Metricbeat and Filebeat when run under the Elastic Agent. {pull}17935[17935] +- Enable introspecting configuration {pull}18124[18124] From 846f0deccaf92bc00626aaaaf334d54c6aafe3f8 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 30 Apr 2020 10:20:12 +0200 Subject: [PATCH 3/6] mage fmt --- .../pkg/agent/application/introspect_config_cmd.go | 3 ++- .../pkg/agent/application/introspect_output_cmd.go | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go b/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go index e63d675a9ef8..56b3f4fda525 100644 --- a/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go @@ -7,11 +7,12 @@ package application import ( "fmt" + yaml "gopkg.in/yaml.v2" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" - yaml "gopkg.in/yaml.v2" ) // IntrospectConfigCmd is an introspect subcommand that shows configurations of the agent. diff --git a/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go b/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go index 4a4966b21414..26eb424fe977 100644 --- a/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go @@ -7,16 +7,17 @@ package application import ( "fmt" + "github.com/urso/ecslog" + "github.com/urso/ecslog/backend" + "github.com/urso/ecslog/backend/appender" + "github.com/urso/ecslog/backend/layout" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/filters" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/plugin/app/monitoring/noop" - "github.com/urso/ecslog" - "github.com/urso/ecslog/backend" - "github.com/urso/ecslog/backend/appender" - "github.com/urso/ecslog/backend/layout" ) // IntrospectOutputCmd is an introspect subcommand that shows configurations of the agent. From fef4aeb5c9c93562f8423e60228c91d3c8702f9c Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 30 Apr 2020 14:57:07 +0200 Subject: [PATCH 4/6] introspect -> inspect --- .../pkg/agent/application/introspect_config_cmd.go | 2 +- x-pack/elastic-agent/pkg/agent/cmd/introspect.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go b/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go index 56b3f4fda525..af590a73448c 100644 --- a/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go @@ -28,7 +28,7 @@ func NewIntrospectConfigCmd(configPath string, }, nil } -// Execute tries to enroll the agent into Fleet. +// Execute introspects agent configuration. func (c *IntrospectConfigCmd) Execute() error { return c.introspectConfig() } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/introspect.go b/x-pack/elastic-agent/pkg/agent/cmd/introspect.go index 6bcd3418cbff..12c5c748253e 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/introspect.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/introspect.go @@ -16,7 +16,7 @@ import ( func newIntrospectCommandWithArgs(flags *globalFlags, s []string, streams *cli.IOStreams) *cobra.Command { cmd := &cobra.Command{ - Use: "introspect", + Use: "inspect", Short: "Shows configuration of the agent", Long: "Shows current configuration of the agent", Args: cobra.ExactArgs(0), From 1e97f9286dee90db98c77427b48736b09b01d0da Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Mon, 4 May 2020 08:38:58 +0200 Subject: [PATCH 5/6] follow -c argument --- x-pack/elastic-agent/pkg/agent/cmd/introspect.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/cmd/introspect.go b/x-pack/elastic-agent/pkg/agent/cmd/introspect.go index 12c5c748253e..f6cb40e18946 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/introspect.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/introspect.go @@ -21,7 +21,7 @@ func newIntrospectCommandWithArgs(flags *globalFlags, s []string, streams *cli.I Long: "Shows current configuration of the agent", Args: cobra.ExactArgs(0), Run: func(c *cobra.Command, args []string) { - command, err := application.NewIntrospectConfigCmd("elastic-agent.yml") + command, err := application.NewIntrospectConfigCmd(flags.Config()) if err != nil { fmt.Fprintf(streams.Err, "%v\n", err) os.Exit(1) @@ -49,7 +49,7 @@ func newIntrospectOutputCommandWithArgs(flags *globalFlags, _ []string, streams outName, _ := c.Flags().GetString("output") program, _ := c.Flags().GetString("program") - command, err := application.NewIntrospectOutputCmd("elastic-agent.yml", outName, program) + command, err := application.NewIntrospectOutputCmd(flags.Config(), outName, program) if err != nil { fmt.Fprintf(streams.Err, "%v\n", err) os.Exit(1) From 833662ad45d9ee8828785a6a5581b05f2d6d2b9d Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Mon, 4 May 2020 18:03:05 +0200 Subject: [PATCH 6/6] conflicts --- .../pkg/agent/application/introspect_config_cmd.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go b/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go index af590a73448c..f5ab634b84de 100644 --- a/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go @@ -9,6 +9,7 @@ import ( yaml "gopkg.in/yaml.v2" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" @@ -77,7 +78,7 @@ func loadFleetConfig(cfg *config.Config) (map[string]interface{}, error) { return nil, err } - as, err := newActionStore(log, storage.NewDiskStore(fleetActionStoreFile())) + as, err := newActionStore(log, storage.NewDiskStore(info.AgentActionStoreFile())) if err != nil { return nil, err }