Merge remote-tracking branch 'upstream/master' into feature/ci-flaky-…

…reporting-changes

* upstream/master: (23 commits)
  [Auditbeat] btmp offset check (elastic#24515)
  Clarify that the Tomcat module is for ingesting access logs (elastic#24543)
  [Ingest Manager] Move logging defaults to agent (elastic#24535)
  Update input-http-endpoint.asciidoc (elastic#24490)
  Fix typo in mqtt input docs (elastic#24509)
  [Elastic Agent] Add verification check when updating communication to Kibana. (elastic#24489)
  Refactor use of system.hostfs to fix cgroup metrics (elastic#24334)
  Add test for close.reader.after_interval to filestream input (elastic#24423)
  chore(ci): use beat_version instead of PR version (elastic#24446)
  Add syntax for multiple selector logging (elastic#24207) (elastic#24497)
  Update Golang to 1.15.9 (elastic#24442)
  [Elastic Agent] Add the ability to provide custom CA's inside of docker. (elastic#24486)
  Add tests for encoding settings of filestream input (elastic#24426)
  [Ingest Manager] Sync on rename on windows (elastic#24504)
  Port four Harvester tests of log input to filestream in Golang (elastic#24250)
  [DOCS] Restructure content for SSL settings (elastic#24342)
  Move example to the correct location in reference docs (elastic#24455)
  Add unit tests for harvester.go of input-logfile package (elastic#24107)
  Fix type for uwsgi.status.worker.rss field (elastic#24468)
  [Ingest Manager] Logging to file disabled on enroll (elastic#24466)
  ...

.ci/packaging.groovy

@@ -429,13 +429,11 @@ def triggerE2ETests(String suite) {
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
     booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
     string(name: 'runTestsSuites', value: suite),
+    string(name: 'BEAT_VERSION', value: env.BEAT_VERSION),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
     string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
     string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
   ]
-  if (isPR()) {
-    parameters.push(string(name: 'BEAT_VERSION', value: "pr-${env.CHANGE_ID}"))
-  }
 
   build(job: "${e2eTestsPipeline}",
     parameters: parameters,

.go-version

@@ -1 +1 @@
-1.15.8
+1.15.9

CHANGELOG-developer.next.asciidoc

@@ -109,3 +109,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Add support for customized monitoring API. {pull}22605[22605]
 - Update Go version to 1.15.7. {pull}22495[22495]
 - Update Go version to 1.15.8. {pull}23955[23955]
+- Update Go version to 1.15.9. {pull}24442[24442]

CHANGELOG.next.asciidoc

@@ -32,6 +32,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update to ECS 1.7.0. {pull}22571[22571]
 - Add support for SCRAM-SHA-512 and SCRAM-SHA-256 in Kafka output. {pull}12867[12867]
 - Fix panic with inline SSL when the certificate or key were small than 256 bytes. {pull}23820[23820]
+- Use alias to report container image in k8s metadata. {pull}24380[24380]
 
 *Auditbeat*
 
@@ -239,6 +240,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix panic due to unhandled DeletedFinalStateUnknown in k8s OnDelete {pull}23419[23419]
 - Fix error loop with runaway CPU use when the Kafka output encounters some connection errors {pull}23484[23484]
 - Allow configuring credential_profile_name and shared_credential_file when using role_arn. {pull}24174[24174]
+- Fix issue discovering docker containers and metadata after reconnections {pull}24318[24318]
+- Allow cgroup self-monitoring to see alternate `hostfs` paths {pull}24334[24334]
 
 *Auditbeat*
 
@@ -261,6 +264,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - system/socket: Having some CPUs unavailable to Auditbeat could cause startup errors or event loss. {pull}22827[22827]
 - Note incompatibility of system/socket on ARM. {pull}23381[23381]
 - system/login: Fixed offset reset on inode reuse. {pull}24414[24414]
+- system/login: Add additional offset check for utmp files. {pull}24515[24515]
 
 *Filebeat*
 
@@ -516,6 +520,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Unskip s3_request integration test. {pull}23887[23887]
 - Add system.hostfs configuration option for system module. {pull}23831[23831]
 - Fix GCP not able to request Cloudfunctions metrics if a region filter was set {pull}24218[24218]
+- Fix type of `uwsgi.status.worker.rss` type. {pull}24468[24468]
 
 *Packetbeat*
 
@@ -1053,6 +1058,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dns.question.top_level_domain fields for sysmon DNS events. {pull}23046[23046]
 - Add Audit and Authentication Polixy Change Events and related.ip information {pull}20684[20684]
 - Add new ECS 1.8 improvements. {pull}23563[23563]
+- Remove deprecated eventlogging api that was used for Windows XP/2003 and associated unused code. {pull}24463[24463]
 
 *Elastic Log Driver*
 

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.8
+FROM golang:1.15.9
 
 RUN \
     apt-get update \

auditbeat/docs/fields.asciidoc

@@ -11970,7 +11970,7 @@ type: keyword
 *`kubernetes.container.name`*::
 +
 --
-Kubernetes container name
+Kubernetes container name (different than the name from the runtime)
 
 
 type: keyword
@@ -11983,7 +11983,9 @@ type: keyword
 Kubernetes container image
 
 
-type: keyword
+type: alias
+
+alias to: container.image.name
 
 --
 

filebeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.8
+FROM golang:1.15.9
 
 RUN \
     apt-get update \

filebeat/docs/fields.asciidoc

@@ -86569,7 +86569,7 @@ type: keyword
 *`kubernetes.container.name`*::
 +
 --
-Kubernetes container name
+Kubernetes container name (different than the name from the runtime)
 
 
 type: keyword
@@ -86582,7 +86582,9 @@ type: keyword
 Kubernetes container image
 
 
-type: keyword
+type: alias
+
+alias to: container.image.name
 
 --
 

filebeat/docs/inputs/input-mqtt.asciidoc

@@ -29,7 +29,7 @@ Example configuration:
 
 <1> `hosts` are required.
 
-<2> `paths` are required.
+<2> `topics` are required.
 
 All other settings are optional.
 

filebeat/docs/modules/tomcat.asciidoc

@@ -12,7 +12,7 @@ This file is generated! See scripts/docs_collector.py
 
 experimental[]
 
-This is a module for receiving Apache Tomcat logs over Syslog or a file.
+This is a module for receiving Apache Tomcat access logs over Syslog or a file.
 
 include::../include/gs-link.asciidoc[]
 

filebeat/input/filestream/config.go

@@ -30,7 +30,7 @@ import (
 
 // Config stores the options of a file stream.
 type config struct {
-	readerConfig
+	Reader readerConfig `config:",inline"`
 
 	Paths          []string                `config:"paths"`
 	Close          closerConfig            `config:"close"`
@@ -79,7 +79,7 @@ type backoffConfig struct {
 
 func defaultConfig() config {
 	return config{
-		readerConfig:   defaultReaderConfig(),
+		Reader:         defaultReaderConfig(),
 		Paths:          []string{},
 		Close:          defaultCloserConfig(),
 		CleanInactive:  0,

filebeat/input/filestream/environment_test.go

@@ -105,17 +105,69 @@ func (e *inputTestingEnvironment) mustWriteLinesToFile(filename string, lines []
 	}
 }
 
+func (e *inputTestingEnvironment) mustAppendLinesToFile(filename string, lines []byte) {
+	path := e.abspath(filename)
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		e.t.Fatalf("failed to open file '%s': %+v", path, err)
+	}
+	defer f.Close()
+
+	_, err = f.Write(lines)
+	if err != nil {
+		e.t.Fatalf("append lines to file '%s': %+v", path, err)
+	}
+}
+
 func (e *inputTestingEnvironment) mustRenameFile(oldname, newname string) {
 	err := os.Rename(e.abspath(oldname), e.abspath(newname))
 	if err != nil {
 		e.t.Fatalf("failed to rename file '%s': %+v", oldname, err)
 	}
 }
 
+func (e *inputTestingEnvironment) mustRemoveFile(filename string) {
+	path := e.abspath(filename)
+	err := os.Remove(path)
+	if err != nil {
+		e.t.Fatalf("failed to rename file '%s': %+v", path, err)
+	}
+}
+
+func (e *inputTestingEnvironment) mustSymlink(filename, symlinkname string) {
+	err := os.Symlink(e.abspath(filename), e.abspath(symlinkname))
+	if err != nil {
+		e.t.Fatalf("failed to create symlink to file '%s': %+v", filename, err)
+	}
+}
+
+func (e *inputTestingEnvironment) mustTruncateFile(filename string, size int64) {
+	path := e.abspath(filename)
+	err := os.Truncate(path, size)
+	if err != nil {
+		e.t.Fatalf("failed to truncate file '%s': %+v", path, err)
+	}
+}
+
 func (e *inputTestingEnvironment) abspath(filename string) string {
 	return filepath.Join(e.workingDir, filename)
 }
 
+func (e *inputTestingEnvironment) requireRegistryEntryCount(expectedCount int) {
+	inputStore, _ := e.stateStore.Access()
+
+	actual := 0
+	err := inputStore.Each(func(_ string, _ statestore.ValueDecoder) (bool, error) {
+		actual += 1
+		return true, nil
+	})
+	if err != nil {
+		e.t.Fatalf("error while iterating through registry: %+v", err)
+	}
+
+	require.Equal(e.t, actual, expectedCount)
+}
+
 // requireOffsetInRegistry checks if the expected offset is set for a file.
 func (e *inputTestingEnvironment) requireOffsetInRegistry(filename string, expectedOffset int) {
 	filepath := e.abspath(filename)
@@ -131,6 +183,32 @@ func (e *inputTestingEnvironment) requireOffsetInRegistry(filename string, expec
 	require.Equal(e.t, expectedOffset, entry.Cursor.Offset)
 }
 
+func (e *inputTestingEnvironment) requireNoEntryInRegistry(filename string) {
+	filepath := e.abspath(filename)
+	fi, err := os.Stat(filepath)
+	if err != nil {
+		e.t.Fatalf("cannot stat file when cheking for offset: %+v", err)
+	}
+
+	inputStore, _ := e.stateStore.Access()
+
+	identifier, _ := newINodeDeviceIdentifier(nil)
+	src := identifier.GetSource(loginp.FSEvent{Info: fi, Op: loginp.OpCreate, NewPath: filepath})
+
+	var entry registryEntry
+	err = inputStore.Get(src.Name(), &entry)
+	if err == nil {
+		e.t.Fatalf("key is not expected to be present '%s'", src.Name())
+	}
+}
+
+// requireOffsetInRegistry checks if the expected offset is set for a file.
+func (e *inputTestingEnvironment) requireOffsetInRegistryByID(key string, expectedOffset int) {
+	entry := e.getRegistryState(key)
+
+	require.Equal(e.t, expectedOffset, entry.Cursor.Offset)
+}
+
 func (e *inputTestingEnvironment) getRegistryState(key string) registryEntry {
 	inputStore, _ := e.stateStore.Access()
 
@@ -153,10 +231,58 @@ func (e *inputTestingEnvironment) waitUntilEventCount(count int) {
 		if sum == count {
 			return
 		}
+		if count < sum {
+			e.t.Fatalf("too many events; expected: %d, actual: %d", count, sum)
+		}
 		time.Sleep(10 * time.Millisecond)
 	}
 }
 
+// waitUntilHarvesterIsDone detects Harvester stop by checking if the last client has been closed
+// as when a Harvester stops the client is closed.
+func (e *inputTestingEnvironment) waitUntilHarvesterIsDone() {
+	for !e.pipeline.clients[len(e.pipeline.clients)-1].closed {
+		time.Sleep(10 * time.Millisecond)
+	}
+}
+
+// requireEventReceived requires that the list of messages has made it into the output.
+func (e *inputTestingEnvironment) requireEventsReceived(events []string) {
+	foundEvents := make([]bool, len(events))
+	checkedEventCount := 0
+	for _, c := range e.pipeline.clients {
+		for _, evt := range c.GetEvents() {
+			if len(events) == checkedEventCount {
+				e.t.Fatalf("not enough expected elements")
+			}
+			message := evt.Fields["message"].(string)
+			if message == events[checkedEventCount] {
+				foundEvents[checkedEventCount] = true
+			}
+			checkedEventCount += 1
+		}
+	}
+
+	var missingEvents []string
+	for i, found := range foundEvents {
+		if !found {
+			missingEvents = append(missingEvents, events[i])
+		}
+	}
+
+	require.Equal(e.t, 0, len(missingEvents), "following events are missing: %+v", missingEvents)
+}
+
+func (e *inputTestingEnvironment) getOutputMessages() []string {
+	messages := make([]string, 0)
+	for _, c := range e.pipeline.clients {
+		for _, evt := range c.GetEvents() {
+			messages = append(messages, evt.Fields["message"].(string))
+		}
+	}
+	return messages
+}
+
 type testInputStore struct {
 	registry *statestore.Registry
 }