Merge branch 'master' into kafka-test-container

elastic · Nov 16, 2021 · efb736f · efb736f
2 parents 4b9095a + 08642d0
commit efb736f
Show file tree

Hide file tree

Showing 53 changed files with 770 additions and 146 deletions.
diff --git a/.mergify.yml b/.mergify.yml
@@ -105,6 +105,8 @@ pull_request_rules:
     conditions:
       - -label~=^backport
       - base=master
+      - -merged
+      - -closed
     actions:
       comment:
         message: |
@@ -120,6 +122,8 @@ pull_request_rules:
   - name: remove-backport label
     conditions:
       - label~=backport-v
+      - -merged
+      - -closed
     actions:
       label:
         remove:

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -138,6 +138,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Periodic metrics in logs will now report `libbeat.output.events.active` and `beat.memstats.rss`
 - Allows disable pod events enrichment with deployment name {pull}28521[28521]
 - Fix `fingerprint` processor to give it access to the `@timestamp` field. {issue}28683[28683]
+- Fix the wrong beat name on monitoring and state endpoint {issue}27755[27755]
 
 *Auditbeat*
 
@@ -180,6 +181,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Relax time parsing and capture group and session type in Cisco ASA module {issue}24710[24710] {pull}28325[28325]
 - Correctly track bytes read when max_bytes is exceeded. {issue}28317[28317] {pull}28352[28352]
 - Upgrade azure-eventhub sdk reference, contains potential checkpoint fixes. {pull}28919[28919]
+- Fix in `aws-s3` input regarding provider discovery through endpoint {pull}28963[28963]
 
 *Heartbeat*
 
@@ -188,6 +190,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Properly locate base stream in fleet configs. {pull}28455[28455]
 - Stop logging params values. {pull}28774[28774]
 - Remove accidentally included cups library in docker images. {pull}28853[pull]
+- Fix broken monitors with newer versions of image relying on dup3. {pull}28938[pull]
 
 *Journalbeat*
 
@@ -234,6 +237,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add source.ip validation for event ID 4778 in the Security module. {issue}19627[19627]
 - Tolerate faults when Windows Event Log session is interrupted {issue}27947[27947] {pull}28191[28191]
 - Add ECS 1.9 new users fields {pull}26509[26509]
+- Don't split hyphenated tokens {pull}28483[28483]
 
 *Functionbeat*
 
@@ -273,6 +277,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update kubernetes scheduler and controllermanager endpoints in elastic-agent-standalone-kubernetes.yaml with secure ports {pull}28675[28675]
 - Add options to configure k8s client qps/burst. {pull}28151[28151]
 - Update to ECS 8.0 fields. {pull}28620[28620]
+- Add http.pprof.enabled option to libbeat to allow http/pprof endpoints on the socket that libbeat creates for metrics. {issue}21965[21965]
+- Support custom analyzers in fields.yml. {issue}28540[28540] {pull}28926[28926]
 
 *Auditbeat*
 
@@ -353,6 +359,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added a new beta `enterprisesearch` module for Elastic Enterprise Search {pull}27549[27549]
 - Preliminary AIX support {pull}27954[27954]
 - Register additional name for `storage` metricset in the azure module. {pull}28447[28447]
+- Update reference to gosigar pacakge for filesystem windows fix. {pull}28909[28909]
 
 *Packetbeat*
 

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -8276,11 +8276,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/gosigar
-Version: v0.14.1
+Version: v0.14.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].2/LICENSE:
 
                               Apache License
                         Version 2.0, January 2004

diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
@@ -1632,6 +1632,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.

diff --git a/docs/devguide/fields-yml.asciidoc b/docs/devguide/fields-yml.asciidoc
@@ -121,3 +121,43 @@ use in aggregations or ordering, you can use a multi-field mapping:
 
 For more information, see the {ref}/multi-fields.html[{es} documentation about
 multi-fields].
+
+==== Defining a text analyzer in-line
+
+It is possible to define a new text analyzer or search analyzer in-line with
+the field definition in the field's mapping parameters.
+
+For example, you can define a new text analyzer that does not break hyphenated names:
+
+[source,yaml]
+----------------------------------------------------------------------
+- key: mybeat
+  title: mybeat
+  description: These are the fields used by mybeat.
+  fields:
+    - name: last_name
+      type: text
+      required: true
+      description: >
+        The last name.
+      analyzer:
+        mybeat_hyphenated_name: <1>
+          type: pattern <2>
+          pattern: "[\\W&&[^-]]+" <3>
+      search_analyzer:
+        mybeat_hyphenated_name: <4>
+          type: pattern
+          pattern: "[\\W&&[^-]]+"
+----------------------------------------------------------------------
+<1> Use a newly defined text analyzer
+<2> Define the custome analyzer type
+<3> Specify the analyzer behaviour
+<4> Use the same analyzer for the search
+
+The names of custom analyzers that are defined in-line may not be reused for a different
+text analyzer. If a text analyzer name is reused it is checked for matching existing
+instances of the analyzer. It is recommended that the analyzer name is prefixed with the
+beat name to avoid name clashes.
+
+For more information, see {ref}/analysis-custom-analyzer.html[{es} documentation about
+defining custom text analyzers].
diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -2544,6 +2544,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.

diff --git a/go.mod b/go.mod
@@ -75,7 +75,7 @@ require (
 	github.com/elastic/go-txfile v0.0.8
 	github.com/elastic/go-ucfg v0.8.3
 	github.com/elastic/go-windows v1.0.1
-	github.com/elastic/gosigar v0.14.1
+	github.com/elastic/gosigar v0.14.2
 	github.com/fatih/color v1.9.0
 	github.com/fearful-symmetry/gorapl v0.0.4
 	github.com/fsnotify/fsevents v0.1.1

diff --git a/go.sum b/go.sum
@@ -537,8 +537,8 @@ github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+F
 github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU=
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
-github.com/elastic/gosigar v0.14.1 h1:T0aQ7n/n2ZA9W7DmAnj60v+qzqKERdBgJBO1CG2W6rc=
-github.com/elastic/gosigar v0.14.1/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
+github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4=
+github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/elastic/sarama v1.19.1-0.20210823122811-11c3ef800752 h1:5/RUNg7rkIvayjPhAIoI3v8p45NfWcfWs5DZSElycis=
 github.com/elastic/sarama v1.19.1-0.20210823122811-11c3ef800752/go.mod h1:mdtqvCSg8JOxk8PmpTNGyo6wzd4BMm4QXSfDnTXmgkE=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=

diff --git a/heartbeat/docs/monitors/monitor-browser.asciidoc b/heartbeat/docs/monitors/monitor-browser.asciidoc
@@ -4,7 +4,7 @@
 TIP: Want to get started with synthetic monitoring?
 See the {observability-guide}/synthetics-quickstart.html[quick start guide].
 
-The options described here configure {beatname_uc} to run the synthetic
+beta[] The options described here configure {beatname_uc} to run the synthetic
 monitoring test suites via Synthetic Agent on the Chromium browser.
 Additional shared options are defined in <<monitor-options>>.
 Example configuration:

diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
@@ -1778,6 +1778,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.

diff --git a/heartbeat/security/security.go b/heartbeat/security/security.go
@@ -147,7 +147,9 @@ func setSeccompRules() error {
 		"close",
 		"connect",
 		"creat",
+		"dup",
 		"dup2",
+		"dup3",
 		"epoll_ctl",
 		"epoll_pwait",
 		"eventfd2",

diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml
@@ -1575,6 +1575,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.

diff --git a/libbeat/_meta/config/http.reference.yml.tmpl b/libbeat/_meta/config/http.reference.yml.tmpl
@@ -22,3 +22,7 @@
 # Descriptor Definition Language (SDDL) to define the permission. This option cannot be used with
 # `http.user`.
 #http.named_pipe.security_descriptor:
+
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
diff --git a/libbeat/api/routes.go b/libbeat/api/routes.go
@@ -20,6 +20,7 @@ package api
 import (
 	"fmt"
 	"net/http"
+	_ "net/http/pprof"
 	"net/url"
 
 	"github.com/elastic/beats/v7/libbeat/common"
@@ -47,6 +48,14 @@ func NewWithDefaultRoutes(log *logp.Logger, config *common.Config, ns lookupFunc
 	return New(log, mux, config)
 }
 
+func (s *Server) AttachPprof() {
+	s.log.Info("Attaching pprof endpoints")
+	s.mux.HandleFunc("/debug/pprof/", func(w http.ResponseWriter, r *http.Request) {
+		http.DefaultServeMux.ServeHTTP(w, r)
+	})
+
+}
+
 func makeRootAPIHandler(handler handlerFunc) handlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path != "/" {

diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go
@@ -105,6 +105,7 @@ type beatConfig struct {
 
 	// beat internal components configurations
 	HTTP            *common.Config         `config:"http"`
+	HTTPPprof       *common.Config         `config:"http.pprof"`
 	Path            paths.Path             `config:"path"`
 	Logging         *common.Config         `config:"logging"`
 	MetricLogging   *common.Config         `config:"logging.metrics"`
@@ -162,19 +163,14 @@ func Run(settings Settings, bt beat.Creator) error {
 		return errw.Wrap(err, "could not set umask")
 	}
 
-	name := settings.Name
-	idxPrefix := settings.IndexPrefix
-	agentVersion := settings.Version
-	elasticLicensed := settings.ElasticLicensed
-
 	return handleError(func() error {
 		defer func() {
 			if r := recover(); r != nil {
-				logp.NewLogger(name).Fatalw("Failed due to panic.",
+				logp.NewLogger(settings.Name).Fatalw("Failed due to panic.",
 					"panic", r, zap.Stack("stack"))
 			}
 		}()
-		b, err := NewBeat(name, idxPrefix, agentVersion, elasticLicensed)
+		b, err := NewInitializedBeat(settings)
 		if err != nil {
 			return err
 		}
@@ -409,10 +405,6 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error {
 	defer logp.Sync()
 	defer logp.Info("%s stopped.", b.Info.Beat)
 
-	err := b.InitWithSettings(settings)
-	if err != nil {
-		return err
-	}
 	defer func() {
 		if err := b.processing.Close(); err != nil {
 			logp.Warn("Failed to close global processing: %v", err)
@@ -428,7 +420,7 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error {
 	// Try to acquire exclusive lock on data path to prevent another beat instance
 	// sharing same data path.
 	bl := newLocker(b)
-	err = bl.lock()
+	err := bl.lock()
 	if err != nil {
 		return err
 	}
@@ -455,6 +447,9 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error {
 		}
 		s.Start()
 		defer s.Stop()
+		if b.Config.HTTPPprof.Enabled() {
+			s.AttachPprof()
+		}
 	}
 
 	if err = seccomp.LoadFilter(b.Config.Seccomp); err != nil {