elastic · narph · Mar 18, 2021 · Jul 23, 2020 · Aug 3, 2020 · Aug 4, 2020
diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -429,13 +429,11 @@ def triggerE2ETests(String suite) {
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
     booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
     string(name: 'runTestsSuites', value: suite),
+    string(name: 'BEAT_VERSION', value: env.BEAT_VERSION),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
     string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
     string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
   ]
-  if (isPR()) {
-    parameters.push(string(name: 'BEAT_VERSION', value: "pr-${env.CHANGE_ID}"))
-  }
 
   build(job: "${e2eTestsPipeline}",
     parameters: parameters,

diff --git a/.go-version b/.go-version
@@ -1 +1 @@
-1.15.8
+1.15.9
diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
@@ -109,3 +109,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Add support for customized monitoring API. {pull}22605[22605]
 - Update Go version to 1.15.7. {pull}22495[22495]
 - Update Go version to 1.15.8. {pull}23955[23955]
+- Update Go version to 1.15.9. {pull}24442[24442]
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -240,6 +240,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix panic due to unhandled DeletedFinalStateUnknown in k8s OnDelete {pull}23419[23419]
 - Fix error loop with runaway CPU use when the Kafka output encounters some connection errors {pull}23484[23484]
 - Allow configuring credential_profile_name and shared_credential_file when using role_arn. {pull}24174[24174]
+- Fix issue discovering docker containers and metadata after reconnections {pull}24318[24318]
+- Allow cgroup self-monitoring to see alternate `hostfs` paths {pull}24334[24334]
 
 *Auditbeat*
 
@@ -518,6 +520,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add system.hostfs configuration option for system module. {pull}23831[23831]
 - Fix GCP not able to request Cloudfunctions metrics if a region filter was set {pull}24218[24218]
 - Fix type of `uwsgi.status.worker.rss` type. {pull}24468[24468]
+- Ignore unsupported derive types for filesystem metricset. {issue}22501[22501] {pull}24502[24502]
 
 *Packetbeat*
 

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -8299,11 +8299,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]/
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/gosigar
-Version: v0.14.0
+Version: v0.14.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE:
 
                               Apache License
                         Version 2.0, January 2004

diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.15.8
+FROM golang:1.15.9
 
 RUN \
     apt-get update \

diff --git a/filebeat/Dockerfile b/filebeat/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.15.8
+FROM golang:1.15.9
 
 RUN \
     apt-get update \

diff --git a/filebeat/docs/inputs/input-mqtt.asciidoc b/filebeat/docs/inputs/input-mqtt.asciidoc
@@ -29,7 +29,7 @@ Example configuration:
 
 <1> `hosts` are required.
 
-<2> `paths` are required.
+<2> `topics` are required.
 
 All other settings are optional.
 

diff --git a/filebeat/docs/modules/tomcat.asciidoc b/filebeat/docs/modules/tomcat.asciidoc
@@ -12,7 +12,7 @@ This file is generated! See scripts/docs_collector.py
 
 experimental[]
 
-This is a module for receiving Apache Tomcat logs over Syslog or a file.
+This is a module for receiving Apache Tomcat access logs over Syslog or a file.
 
 include::../include/gs-link.asciidoc[]
 

diff --git a/filebeat/input/filestream/config.go b/filebeat/input/filestream/config.go
@@ -30,7 +30,7 @@ import (
 
 // Config stores the options of a file stream.
 type config struct {
-	readerConfig
+	Reader readerConfig `config:",inline"`
 
 	Paths          []string                `config:"paths"`
 	Close          closerConfig            `config:"close"`
@@ -79,7 +79,7 @@ type backoffConfig struct {
 
 func defaultConfig() config {
 	return config{
-		readerConfig:   defaultReaderConfig(),
+		Reader:         defaultReaderConfig(),
 		Paths:          []string{},
 		Close:          defaultCloserConfig(),
 		CleanInactive:  0,

diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go
@@ -53,14 +53,8 @@ type fileMeta struct {
 // are actively written by other applications.
 type filestream struct {
 	readerConfig    readerConfig
-	bufferSize      int
-	tailFile        bool // TODO
 	encodingFactory encoding.EncodingFactory
 	encoding        encoding.Encoding
-	lineTerminator  readfile.LineTerminator
-	excludeLines    []match.Matcher
-	includeLines    []match.Matcher
-	maxBytes        int
 	closerConfig    closerConfig
 }
 
@@ -97,9 +91,9 @@ func configure(cfg *common.Config) (loginp.Prospector, loginp.Harvester, error)
 		return nil, nil, fmt.Errorf("error while creating file identifier: %v", err)
 	}
 
-	encodingFactory, ok := encoding.FindEncoding(config.Encoding)
+	encodingFactory, ok := encoding.FindEncoding(config.Reader.Encoding)
 	if !ok || encodingFactory == nil {
-		return nil, nil, fmt.Errorf("unknown encoding('%v')", config.Encoding)
+		return nil, nil, fmt.Errorf("unknown encoding('%v')", config.Reader.Encoding)
 	}
 
 	prospector := &fileProspector{
@@ -111,13 +105,8 @@ func configure(cfg *common.Config) (loginp.Prospector, loginp.Harvester, error)
 	}
 
 	filestream := &filestream{
-		readerConfig:    config.readerConfig,
-		bufferSize:      config.BufferSize,
+		readerConfig:    config.Reader,
 		encodingFactory: encodingFactory,
-		lineTerminator:  config.LineTerminator,
-		excludeLines:    config.ExcludeLines,
-		includeLines:    config.IncludeLines,
-		maxBytes:        config.MaxBytes,
 		closerConfig:    config.Close,
 	}
 
@@ -191,7 +180,7 @@ func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, path stri
 		return nil, err
 	}
 
-	log.Debug("newLogFileReader with config.MaxBytes:", inp.maxBytes)
+	log.Debug("newLogFileReader with config.MaxBytes:", inp.readerConfig.MaxBytes)
 
 	// TODO: NewLineReader uses additional buffering to deal with encoding and testing
 	//       for new lines in input stream. Simple 8-bit based encodings, or plain
@@ -211,22 +200,22 @@ func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, path stri
 	// for the worst case scenario where incoming UTF32 charchers are decoded to the single byte UTF-8 characters.
 	// This limit serves primarily to avoid memory bload or potential OOM with expectedly long lines in the file.
 	// The further size limiting is performed by LimitReader at the end of the readers pipeline as needed.
-	encReaderMaxBytes := inp.maxBytes * 4
+	encReaderMaxBytes := inp.readerConfig.MaxBytes * 4
 
 	var r reader.Reader
 	r, err = readfile.NewEncodeReader(dbgReader, readfile.Config{
 		Codec:      inp.encoding,
-		BufferSize: inp.bufferSize,
-		Terminator: inp.lineTerminator,
+		BufferSize: inp.readerConfig.BufferSize,
+		Terminator: inp.readerConfig.LineTerminator,
 		MaxBytes:   encReaderMaxBytes,
 	})
 	if err != nil {
 		f.Close()
 		return nil, err
 	}
 
-	r = readfile.NewStripNewline(r, inp.lineTerminator)
-	r = readfile.NewLimitReader(r, inp.maxBytes)
+	r = readfile.NewStripNewline(r, inp.readerConfig.LineTerminator)
+	r = readfile.NewLimitReader(r, inp.readerConfig.MaxBytes)
 
 	return r, nil
 }
@@ -335,14 +324,14 @@ func (inp *filestream) readFromSource(
 // isDroppedLine decides if the line is exported or not based on
 // the include_lines and exclude_lines options.
 func (inp *filestream) isDroppedLine(log *logp.Logger, line string) bool {
-	if len(inp.includeLines) > 0 {
-		if !matchAny(inp.includeLines, line) {
+	if len(inp.readerConfig.IncludeLines) > 0 {
+		if !matchAny(inp.readerConfig.IncludeLines, line) {
 			log.Debug("Drop line as it does not match any of the include patterns %s", line)
 			return true
 		}
 	}
-	if len(inp.excludeLines) > 0 {
-		if matchAny(inp.excludeLines, line) {
+	if len(inp.readerConfig.ExcludeLines) > 0 {
+		if matchAny(inp.readerConfig.ExcludeLines, line) {
 			log.Debug("Drop line as it does match one of the exclude patterns%s", line)
 			return true
 		}

diff --git a/filebeat/input/filestream/input_integration_test.go b/filebeat/input/filestream/input_integration_test.go
@@ -20,11 +20,17 @@
 package filestream
 
 import (
+	"bytes"
 	"context"
 	"os"
 	"runtime"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
+
 	loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile"
 )
 
@@ -201,3 +207,107 @@ func TestFilestreamEmptyLinesOnly(t *testing.T) {
 
 	env.requireNoEntryInRegistry(testlogName)
 }
+
+// test_bom_utf8 from test_harvester.py
+func TestFilestreamBOMUTF8(t *testing.T) {
+	env := newInputTestingEnvironment(t)
+
+	testlogName := "test.log"
+	inp := env.mustCreateInput(map[string]interface{}{
+		"paths": []string{env.abspath(testlogName)},
+	})
+
+	ctx, cancelInput := context.WithCancel(context.Background())
+	env.startInput(ctx, inp)
+
+	// BOM: 0xEF,0xBB,0xBF
+	lines := append([]byte{0xEF, 0xBB, 0xBF}, []byte(`#Software: Microsoft Exchange Server
+#Version: 14.0.0.0
+#Log-type: Message Tracking Log
+#Date: 2016-04-05T00:00:02.052Z
+#Fields: date-time,client-ip,client-hostname,server-ip,server-hostname,source-context,connector-id,source,event-id,internal-message-id,message-id,recipient-address,recipient-status,total-bytes,recipient-count,related-recipient-address,reference,message-subject,sender-address,return-path,message-info,directionality,tenant-id,original-client-ip,original-server-ip,custom-data
+2016-04-05T00:00:02.052Z,,,,,"MDB:61914740-3f1b-4ddb-94e0-557196870cfa, Mailbox:279f077c-216f-4323-a9ee-48e50ffd3cad, Event:269492708, MessageClass:IPM.Note.StorageQuotaWarning.Warning, CreationTime:2016-04-05T00:00:01.022Z, ClientType:System",,STOREDRIVER,NOTIFYMAPI,,,,,,,,,,,,,,,,,S:ItemEntryId=00-00-00-00-37-DB-F9-F9-B5-F2-42-4F-86-62-E6-5D-FC-0C-A1-41-07-00-0E-D6-03-16-80-DC-8C-44-9D-30-07-23-ED-71-B7-F7-00-00-1F-D4-B5-0E-00-00-2E-EF-F2-59-0E-E8-2D-46-BC-31-02-85-0D-67-98-43-00-00-37-4A-A3-B3-00-00
+2016-04-05T00:00:02.145Z,,,,,"MDB:61914740-3f1b-4ddb-94e0-557196870cfa, Mailbox:49cb09c6-5b76-415d-a085-da0ad9079682, Event:269492711, MessageClass:IPM.Note.StorageQuotaWarning.Warning, CreationTime:2016-04-05T00:00:01.038Z, ClientType:System",,STOREDRIVER,NOTIFYMAPI,,,,,,,,,,,,,,,,,S:ItemEntryId=00-00-00-00-97-8F-07-43-51-44-61-4A-AD-BD-29-D4-97-4E-20-A0-07-00-0E-D6-03-16-80-DC-8C-44-9D-30-07-23-ED-71-B7-F7-00-8E-8F-BD-EB-57-00-00-3D-FB-CE-26-A4-8D-46-4C-A4-35-0F-A7-9B-FA-D7-B9-00-00-37-44-2F-CA-00-00
+`)...)
+	env.mustWriteLinesToFile(testlogName, lines)
+
+	env.waitUntilEventCount(7)
+
+	cancelInput()
+	env.waitUntilInputStops()
+
+	messages := env.getOutputMessages()
+	require.Equal(t, messages[0], "#Software: Microsoft Exchange Server")
+}
+
+// test_boms from test_harvester.py
+func TestFilestreamUTF16BOMs(t *testing.T) {
+	encodings := map[string]encoding.Encoding{
+		"utf-16be-bom": unicode.UTF16(unicode.BigEndian, unicode.UseBOM),
+		"utf-16le-bom": unicode.UTF16(unicode.LittleEndian, unicode.UseBOM),
+	}
+
+	for name, enc := range encodings {
+		name := name
+		encoder := enc.NewEncoder()
+		t.Run(name, func(t *testing.T) {
+			env := newInputTestingEnvironment(t)
+
+			testlogName := "test.log"
+			inp := env.mustCreateInput(map[string]interface{}{
+				"paths":    []string{env.abspath(testlogName)},
+				"encoding": name,
+			})
+
+			ctx, cancelInput := context.WithCancel(context.Background())
+			env.startInput(ctx, inp)
+
+			line := []byte("first line\n")
+			buf := bytes.NewBuffer(nil)
+			writer := transform.NewWriter(buf, encoder)
+			writer.Write(line)
+			writer.Close()
+
+			env.mustWriteLinesToFile(testlogName, buf.Bytes())
+
+			env.waitUntilEventCount(1)
+
+			env.requireEventsReceived([]string{"first line"})
+
+			cancelInput()
+			env.waitUntilInputStops()
+		})
+	}
+}
+
+// test_close_timeout from test_harvester.py
+func TestFilestreamCloseTimeout(t *testing.T) {
+	env := newInputTestingEnvironment(t)
+
+	testlogName := "test.log"
+	inp := env.mustCreateInput(map[string]interface{}{
+		"paths":                                []string{env.abspath(testlogName)},
+		"prospector.scanner.check_interval":    "24h",
+		"close.on_state_change.check_interval": "100ms",
+		"close.reader.after_interval":          "500ms",
+	})
+
+	testlines := []byte("first line\n")
+	env.mustWriteLinesToFile(testlogName, testlines)
+
+	ctx, cancelInput := context.WithCancel(context.Background())
+	env.startInput(ctx, inp)
+
+	env.waitUntilEventCount(1)
+	env.requireOffsetInRegistry(testlogName, len(testlines))
+	env.waitUntilHarvesterIsDone()
+
+	env.mustWriteLinesToFile(testlogName, []byte("first line\nsecond log line\n"))
+
+	env.waitUntilEventCount(1)
+
+	cancelInput()
+	env.waitUntilInputStops()
+
+	env.requireOffsetInRegistry(testlogName, len(testlines))
+}
diff --git a/go.mod b/go.mod
@@ -73,7 +73,7 @@ require (
 	github.com/elastic/go-txfile v0.0.7
 	github.com/elastic/go-ucfg v0.8.3
 	github.com/elastic/go-windows v1.0.1 // indirect
-	github.com/elastic/gosigar v0.14.0
+	github.com/elastic/gosigar v0.14.1
 	github.com/fatih/color v1.9.0
 	github.com/fsnotify/fsevents v0.1.1
 	github.com/fsnotify/fsnotify v1.4.9

diff --git a/go.sum b/go.sum
@@ -278,8 +278,8 @@ github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+F
 github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU=
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
-github.com/elastic/gosigar v0.14.0 h1:5w470Q8AagzVY8U48ab8rVkQrOXiIK1NHBYWrAxi9kI=
-github.com/elastic/gosigar v0.14.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
+github.com/elastic/gosigar v0.14.1 h1:T0aQ7n/n2ZA9W7DmAnj60v+qzqKERdBgJBO1CG2W6rc=
+github.com/elastic/gosigar v0.14.1/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/elastic/sarama v1.19.1-0.20210120173147-5c8cb347d877 h1:C9LsbipColsz04JKpKoLlp0pgMJRLq2uXVTeKRDcNcY=
 github.com/elastic/sarama v1.19.1-0.20210120173147-5c8cb347d877/go.mod h1:g5s5osgELxgM+Md9Qni9rzo7Rbt+vvFQI4bt/Mc93II=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=

diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.15.8
+FROM golang:1.15.9
 
 RUN \
     apt-get update \

diff --git a/journalbeat/Dockerfile b/journalbeat/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.15.8
+FROM golang:1.15.9
 
 RUN \
     apt-get update \

diff --git a/libbeat/Dockerfile b/libbeat/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.15.8
+FROM golang:1.15.9
 
 RUN \
     apt-get update \

diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go
@@ -44,6 +44,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/cloudid"
 	"github.com/elastic/beats/v7/libbeat/cmd/instance/metrics"
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/v7/libbeat/common/file"
 	"github.com/elastic/beats/v7/libbeat/common/reload"
 	"github.com/elastic/beats/v7/libbeat/common/seccomp"
@@ -1083,13 +1084,22 @@ func initPaths(cfg *common.Config) error {
 	// the paths field. After we will unpack the complete configuration and keystore reference
 	// will be correctly replaced.
 	partialConfig := struct {
-		Path paths.Path `config:"path"`
+		Path   paths.Path `config:"path"`
+		Hostfs string     `config:"system.hostfs"`
 	}{}
 
+	if paths.IsCLISet() {
+		cfgwarn.Deprecate("8.0.0", "This flag will be removed in the future and replaced by a config value.")
+	}
+
 	if err := cfg.Unpack(&partialConfig); err != nil {
 		return fmt.Errorf("error extracting default paths: %+v", err)
 	}
 
+	// Read the value for hostfs as `system.hostfs`
+	// In the config, there is no `path.hostfs`, as we're merely using the path struct to carry the hostfs variable.
+	partialConfig.Path.Hostfs = partialConfig.Hostfs
+
 	if err := paths.InitPaths(&partialConfig.Path); err != nil {
 		return fmt.Errorf("error setting default paths: %+v", err)
 	}