From 88a0b3ab5749aa54efc0f7f5b16987f8949b0bf7 Mon Sep 17 00:00:00 2001
From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com>
Date: Mon, 15 Nov 2021 12:22:25 -0800
Subject: [PATCH] Add config option to attach pprof endpoints to http socket
 (#28902)

* Add config option to attach pprof endpoints to http socket

* Fix linting issues

* add security note

* Add pprof security docs

(cherry picked from commit 3d542914ce0af93c99d4bf719bf8498e3b7d6f83)
---
 CHANGELOG.next.asciidoc                       |  1 +
 auditbeat/auditbeat.reference.yml             |  4 ++
 filebeat/filebeat.reference.yml               |  4 ++
 heartbeat/heartbeat.reference.yml             |  4 ++
 journalbeat/journalbeat.reference.yml         |  4 ++
 libbeat/_meta/config/http.reference.yml.tmpl  |  4 ++
 libbeat/api/routes.go                         |  9 +++++
 libbeat/cmd/instance/beat.go                  |  4 ++
 libbeat/docs/http-endpoint.asciidoc           |  1 +
 libbeat/tests/system/test_http.py             |  7 ++++
 libbeat/tests/system/test_http_pprof.py       | 39 +++++++++++++++++++
 metricbeat/metricbeat.reference.yml           |  4 ++
 packetbeat/packetbeat.reference.yml           |  4 ++
 winlogbeat/winlogbeat.reference.yml           |  4 ++
 x-pack/auditbeat/auditbeat.reference.yml      |  4 ++
 x-pack/filebeat/filebeat.reference.yml        |  4 ++
 .../functionbeat/functionbeat.reference.yml   |  4 ++
 x-pack/heartbeat/heartbeat.reference.yml      |  4 ++
 x-pack/metricbeat/metricbeat.reference.yml    |  4 ++
 x-pack/osquerybeat/osquerybeat.reference.yml  |  4 ++
 x-pack/packetbeat/packetbeat.reference.yml    |  4 ++
 x-pack/winlogbeat/winlogbeat.reference.yml    |  4 ++
 22 files changed, 125 insertions(+)
 create mode 100644 libbeat/tests/system/test_http_pprof.py

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 90c87bd4059..bcfd024792c 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -266,6 +266,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update kubernetes scheduler and controllermanager endpoints in elastic-agent-standalone-kubernetes.yaml with secure ports {pull}28675[28675]
 - Add options to configure k8s client qps/burst. {pull}28151[28151]
 - Update to ECS 8.0 fields. {pull}28620[28620]
+- Add http.pprof.enabled option to libbeat to allow http/pprof endpoints on the socket that libbeat creates for metrics. {issue}21965[21965]
 
 *Auditbeat*
 
diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
index a6adc66ac34..3d1435314b4 100644
--- a/auditbeat/auditbeat.reference.yml
+++ b/auditbeat/auditbeat.reference.yml
@@ -1632,6 +1632,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
index 9ddcc148329..dc7250e402c 100644
--- a/filebeat/filebeat.reference.yml
+++ b/filebeat/filebeat.reference.yml
@@ -2544,6 +2544,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
index e3b07c015de..1eb37188303 100644
--- a/heartbeat/heartbeat.reference.yml
+++ b/heartbeat/heartbeat.reference.yml
@@ -1778,6 +1778,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml
index f0a85bd09eb..5058ed19cce 100644
--- a/journalbeat/journalbeat.reference.yml
+++ b/journalbeat/journalbeat.reference.yml
@@ -1575,6 +1575,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/libbeat/_meta/config/http.reference.yml.tmpl b/libbeat/_meta/config/http.reference.yml.tmpl
index 19a9f5fcd50..ccf85bb6189 100644
--- a/libbeat/_meta/config/http.reference.yml.tmpl
+++ b/libbeat/_meta/config/http.reference.yml.tmpl
@@ -22,3 +22,7 @@
 # Descriptor Definition Language (SDDL) to define the permission. This option cannot be used with
 # `http.user`.
 #http.named_pipe.security_descriptor:
+
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
diff --git a/libbeat/api/routes.go b/libbeat/api/routes.go
index bc72347cf28..14b213b1928 100644
--- a/libbeat/api/routes.go
+++ b/libbeat/api/routes.go
@@ -20,6 +20,7 @@ package api
 import (
 	"fmt"
 	"net/http"
+	_ "net/http/pprof"
 	"net/url"
 
 	"github.com/elastic/beats/v7/libbeat/common"
@@ -47,6 +48,14 @@ func NewWithDefaultRoutes(log *logp.Logger, config *common.Config, ns lookupFunc
 	return New(log, mux, config)
 }
 
+func (s *Server) AttachPprof() {
+	s.log.Info("Attaching pprof endpoints")
+	s.mux.HandleFunc("/debug/pprof/", func(w http.ResponseWriter, r *http.Request) {
+		http.DefaultServeMux.ServeHTTP(w, r)
+	})
+
+}
+
 func makeRootAPIHandler(handler handlerFunc) handlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path != "/" {
diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go
index 9b4d05494f5..e5186ea5944 100644
--- a/libbeat/cmd/instance/beat.go
+++ b/libbeat/cmd/instance/beat.go
@@ -105,6 +105,7 @@ type beatConfig struct {
 
 	// beat internal components configurations
 	HTTP            *common.Config         `config:"http"`
+	HTTPPprof       *common.Config         `config:"http.pprof"`
 	Path            paths.Path             `config:"path"`
 	Logging         *common.Config         `config:"logging"`
 	MetricLogging   *common.Config         `config:"logging.metrics"`
@@ -455,6 +456,9 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error {
 		}
 		s.Start()
 		defer s.Stop()
+		if b.Config.HTTPPprof.Enabled() {
+			s.AttachPprof()
+		}
 	}
 
 	if err = seccomp.LoadFilter(b.Config.Seccomp); err != nil {
diff --git a/libbeat/docs/http-endpoint.asciidoc b/libbeat/docs/http-endpoint.asciidoc
index 0db4e705935..853e7d3c2d9 100644
--- a/libbeat/docs/http-endpoint.asciidoc
+++ b/libbeat/docs/http-endpoint.asciidoc
@@ -32,6 +32,7 @@ It is recommended to use only localhost. Default is `localhost`
 current user.
 `http.named_pipe.security_descriptor`:: (Optional) Windows Security descriptor string defined in the SDDL format. Default to
 read and write permission for the current user.
+`http.pprof.enabled`:: (Optional) Enable the `/debug/pprof/` endpoints when serving HTTP. It is recommended that this is only enabled on localhost as these endpoints may leak data. Default is `false`.
 
 This is the list of paths you can access. For pretty JSON output append `?pretty` to the URL.
 
diff --git a/libbeat/tests/system/test_http.py b/libbeat/tests/system/test_http.py
index 5c1baa81bd2..76e5b40b181 100644
--- a/libbeat/tests/system/test_http.py
+++ b/libbeat/tests/system/test_http.py
@@ -47,3 +47,10 @@ def test_error(self):
         """
         r = requests.get("http://localhost:5066/not-exist")
         assert r.status_code == 404
+
+    def test_pprof_disabled(self):
+        """
+        Test /debug/pprof/ http endpoint
+        """
+        r = requests.get("http://localhost:5066/debug/pprof/")
+        assert r.status_code == 404
diff --git a/libbeat/tests/system/test_http_pprof.py b/libbeat/tests/system/test_http_pprof.py
new file mode 100644
index 00000000000..0276206fc9a
--- /dev/null
+++ b/libbeat/tests/system/test_http_pprof.py
@@ -0,0 +1,39 @@
+from base import BaseTest
+
+import requests
+import json
+
+
+class Test(BaseTest):
+    def setUp(self):
+        super(BaseTest, self).setUp()
+        self.render_config_template()
+        self.proc = self.start_beat(extra_args=["-E", "http.enabled=true", "-E", "http.pprof.enabled=true"])
+        self.wait_until(lambda: self.log_contains("Starting stats endpoint"))
+
+    def tearDown(self):
+        super(BaseTest, self).tearDown()
+        # Wait till the beat is completely started so it can handle SIGTERM
+        self.wait_until(lambda: self.log_contains("mockbeat start running."))
+        self.proc.check_kill_and_wait()
+
+    def test_pprof(self):
+        """
+        Test /debug/pprof/ http endpoint
+        """
+        r = requests.get("http://localhost:5066/debug/pprof/")
+        assert r.status_code == 200
+
+    def test_pprof_cmdline(self):
+        """
+        Test /debug/pprof/cmdline http endpoint
+        """
+        r = requests.get("http://localhost:5066/debug/pprof/cmdline")
+        assert r.status_code == 200
+
+    def test_pprof_error(self):
+        """
+        Test not existing http endpoint
+        """
+        r = requests.get("http://localhost:5066/debug/pprof/not-exist")
+        assert r.status_code == 404
diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
index c8381674854..746c04d54ed 100644
--- a/metricbeat/metricbeat.reference.yml
+++ b/metricbeat/metricbeat.reference.yml
@@ -2447,6 +2447,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml
index 2875560800a..3377abc029a 100644
--- a/packetbeat/packetbeat.reference.yml
+++ b/packetbeat/packetbeat.reference.yml
@@ -2127,6 +2127,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml
index 3831d25a6fe..77886958368 100644
--- a/winlogbeat/winlogbeat.reference.yml
+++ b/winlogbeat/winlogbeat.reference.yml
@@ -1555,6 +1555,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml
index 8f77c76ee14..bab59c339c1 100644
--- a/x-pack/auditbeat/auditbeat.reference.yml
+++ b/x-pack/auditbeat/auditbeat.reference.yml
@@ -1688,6 +1688,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
index f45a0ab851d..428230017f7 100644
--- a/x-pack/filebeat/filebeat.reference.yml
+++ b/x-pack/filebeat/filebeat.reference.yml
@@ -4697,6 +4697,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml
index 1c68196633a..4b6dab7943f 100644
--- a/x-pack/functionbeat/functionbeat.reference.yml
+++ b/x-pack/functionbeat/functionbeat.reference.yml
@@ -1426,6 +1426,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml
index e3b07c015de..1eb37188303 100644
--- a/x-pack/heartbeat/heartbeat.reference.yml
+++ b/x-pack/heartbeat/heartbeat.reference.yml
@@ -1778,6 +1778,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
index 539fcea1baa..77773b8ece7 100644
--- a/x-pack/metricbeat/metricbeat.reference.yml
+++ b/x-pack/metricbeat/metricbeat.reference.yml
@@ -2968,6 +2968,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml
index 945cfe9ce78..13dfcc2089e 100644
--- a/x-pack/osquerybeat/osquerybeat.reference.yml
+++ b/x-pack/osquerybeat/osquerybeat.reference.yml
@@ -1145,6 +1145,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml
index 2875560800a..3377abc029a 100644
--- a/x-pack/packetbeat/packetbeat.reference.yml
+++ b/x-pack/packetbeat/packetbeat.reference.yml
@@ -2127,6 +2127,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml
index efe36dec031..09afd2e6208 100644
--- a/x-pack/winlogbeat/winlogbeat.reference.yml
+++ b/x-pack/winlogbeat/winlogbeat.reference.yml
@@ -1598,6 +1598,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.