From 471acb6d791c13ceb8a4a25313f80b87a6dc7f2c Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Mon, 22 Nov 2021 12:59:14 -0800 Subject: [PATCH] Enable pprof for elastic-agent and beats (#28983) Enable the /debug/pprof/ endpoints for all beats that the elastic-agent starts. Enable the pprof endpoints on elastic-agent if agent.monitoring.pprof is true (default true). Agent endpoint can be toggled in case it is located on a network and not localhost/unix socket/windows N pipe. (cherry picked from commit 6ad6bee11da2fa1243491cdfde5c235de8a971bb) --- x-pack/elastic-agent/CHANGELOG.next.asciidoc | 1 + x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl | 3 +++ .../elastic-agent/_meta/config/common.reference.p2.yml.tmpl | 3 +++ .../elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl | 3 +++ x-pack/elastic-agent/elastic-agent.docker.yml | 3 +++ x-pack/elastic-agent/elastic-agent.reference.yml | 3 +++ x-pack/elastic-agent/elastic-agent.yml | 3 +++ x-pack/elastic-agent/pkg/agent/cmd/run.go | 4 ++++ .../elastic-agent/pkg/core/monitoring/beats/beats_monitor.go | 1 + x-pack/elastic-agent/pkg/core/monitoring/config/config.go | 2 ++ 10 files changed, 26 insertions(+) diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 4ebf989b89be..d0de29fdea8e 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -149,3 +149,4 @@ - Add diagnostics command to gather beat metadata. {pull}28265[28265] - Add diagnostics collect command to gather beat metadata, config, policy, and logs and bundle it into an archive. {pull}28461[28461] - Add `KIBANA_FLEET_SERVICE_TOKEN` to Elastic Agent container. {pull}28096[28096] +- Enable pprof endpoints for beats processes. Allow pprof endpoints for elastic-agent if enabled. {pull}28983[28983] diff --git a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl index 1850c003f46d..23d6203c2ecc 100644 --- a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl @@ -33,6 +33,9 @@ inputs: # logs: true # # enables metrics monitoring # metrics: true +# # exposes /debug/pprof/ endpoints +# # recommended that these endpoints are only enabled if the monitoring endpoint is set to localhost +# pprof: true # # exposes agent metrics using http, by default sockets and named pipes are used # http: # # enables http endpoint diff --git a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl index f7593c383ac8..312730db5920 100644 --- a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl @@ -107,6 +107,9 @@ inputs: # logs: false # # enables metrics monitoring # metrics: false +# # exposes /debug/pprof/ endpoints +# # recommended that these endpoints are only enabled if the monitoring endpoint is set to localhost +# pprof: true # # exposes agent metrics using http, by default sockets and named pipes are used # http: # # enables http endpoint diff --git a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl index 0b73c9fcdb76..78c64b51494b 100644 --- a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl @@ -107,6 +107,9 @@ inputs: # logs: false # # enables metrics monitoring # metrics: false +# # exposes /debug/pprof/ endpoints +# # recommended that these endpoints are only enabled if the monitoring endpoint is set to localhost +# pprof: true # # exposes agent metrics using http, by default sockets and named pipes are used # http: # # enables http endpoint diff --git a/x-pack/elastic-agent/elastic-agent.docker.yml b/x-pack/elastic-agent/elastic-agent.docker.yml index cb77ab59ffc9..81b48af034dd 100644 --- a/x-pack/elastic-agent/elastic-agent.docker.yml +++ b/x-pack/elastic-agent/elastic-agent.docker.yml @@ -107,6 +107,9 @@ inputs: # logs: false # # enables metrics monitoring # metrics: false +# # exposes /debug/pprof/ endpoints +# # recommended that these endpoints are only enabled if the monitoring endpoint is set to localhost +# pprof: true # # exposes agent metrics using http, by default sockets and named pipes are used # http: # # enables http endpoint diff --git a/x-pack/elastic-agent/elastic-agent.reference.yml b/x-pack/elastic-agent/elastic-agent.reference.yml index 5456097ce3a3..e0c4077f282d 100644 --- a/x-pack/elastic-agent/elastic-agent.reference.yml +++ b/x-pack/elastic-agent/elastic-agent.reference.yml @@ -113,6 +113,9 @@ inputs: # logs: false # # enables metrics monitoring # metrics: false +# # exposes /debug/pprof/ endpoints +# # recommended that these endpoints are only enabled if the monitoring endpoint is set to localhost +# pprof: true # # exposes agent metrics using http, by default sockets and named pipes are used # http: # # enables http endpoint diff --git a/x-pack/elastic-agent/elastic-agent.yml b/x-pack/elastic-agent/elastic-agent.yml index 8204503be516..70926508570e 100644 --- a/x-pack/elastic-agent/elastic-agent.yml +++ b/x-pack/elastic-agent/elastic-agent.yml @@ -39,6 +39,9 @@ inputs: # logs: true # # enables metrics monitoring # metrics: true +# # exposes /debug/pprof/ endpoints +# # recommended that these endpoints are only enabled if the monitoring endpoint is set to localhost +# pprof: true # # exposes agent metrics using http, by default sockets and named pipes are used # http: # # enables http endpoint diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index 56a41e068957..1c8c1dd4916f 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -313,6 +313,10 @@ func setupMetrics(agentInfo *info.AgentInfo, logger *logger.Logger, operatingSys } s.Start() + if cfg.Pprof { + s.AttachPprof() + } + // return server stopper return s.Stop, nil } diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go index e00860ec1f10..d506c22a6cba 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go @@ -122,6 +122,7 @@ func (b *Monitor) EnrichArgs(spec program.Spec, pipelineID string, args []string appendix = append(appendix, "-E", "http.enabled=true", "-E", "http.host="+endpoint, + "-E", "http.pprof.enabled=true", ) } diff --git a/x-pack/elastic-agent/pkg/core/monitoring/config/config.go b/x-pack/elastic-agent/pkg/core/monitoring/config/config.go index 7e1dbc77273a..10a3a6bd4a91 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/config/config.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/config/config.go @@ -15,6 +15,7 @@ type MonitoringConfig struct { LogMetrics bool `yaml:"-" config:"-"` HTTP *MonitoringHTTPConfig `yaml:"http" config:"http"` Namespace string `yaml:"namespace" config:"namespace"` + Pprof bool `yaml:"pprof" config:"pprof"` } // MonitoringHTTPConfig is a config defining HTTP endpoint published by agent @@ -38,5 +39,6 @@ func DefaultConfig() *MonitoringConfig { Port: defaultPort, }, Namespace: defaultNamespace, + Pprof: true, } }