Merge remote-tracking branch 'upstream/master' into github-actions-build

* upstream/master: [libbeat] Fix add_labels flattening of arrays values (elastic#29211) Change elastic-agent pprof default to false (elastic#29155) elastic#28472 fix flaky tests in libbeat fmtstr to use time.UTC instead of time.Local (elastic#28473) Adopt `parsers` in Filebeat's journald input (elastic#29070) [Elastic Agent] Add process error handling guidelines (elastic#29152) winlogbeat/sys/winevent: use reflect IsZero method (elastic#29190) Remove Journalbeat (elastic#29131) Add note that there is no warranty or support for generator code (elastic#28797) packetbeat: preparation for npcap addition (elastic#29017) Use the generic helper for opening file to read in filestream (elastic#29180) Workflow for macos (elastic#29174) Fix `decode_json_fields` processor to always add error key (elastic#29107)
nxei · Dec 1, 2021 · 99c50fb · 99c50fb
2 parents bc05e65 + 1dd9714
commit 99c50fb
Show file tree

Hide file tree

Showing 216 changed files with 1,456 additions and 22,662 deletions.
diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -101,7 +101,6 @@ pipeline {
                   'auditbeat',
                   'filebeat',
                   'heartbeat',
-                  'journalbeat',
                   'metricbeat',
                   'packetbeat',
                   'winlogbeat',
@@ -111,7 +110,6 @@ pipeline {
                   'x-pack/filebeat',
                   'x-pack/functionbeat',
                    'x-pack/heartbeat',
-                  // 'x-pack/journalbeat',
                   'x-pack/metricbeat',
                   'x-pack/osquerybeat',
                   'x-pack/packetbeat',
@@ -199,7 +197,6 @@ pipeline {
                   'auditbeat',
                   'filebeat',
                   'heartbeat',
-                  'journalbeat',
                   'metricbeat',
                   'packetbeat',
                   'x-pack/auditbeat',
@@ -277,8 +274,6 @@ def pushCIDockerImages(Map args = [:]) {
       tagAndPush(beatName: 'filebeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('heartbeat')) {
       tagAndPush(beatName: 'heartbeat', arch: arch)
-    } else if ("${env.BEATS_FOLDER}" == "journalbeat"){
-      tagAndPush(beatName: 'journalbeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) {
       tagAndPush(beatName: 'metricbeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('osquerybeat')) {

diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml
@@ -1,7 +1,6 @@
 name: macos-build
 env:
   GOPATH: /home/runner/work/beats/beats
-  GO111MODULE: off
   REPO_PATH: src/beats
   MACOS_GOPATH: /Users/runner/work/beats/beats
   GITHUB_TOKEN: ${{ github.token }}

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -28,6 +28,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Index template's default_fields setting is only populated with ECS fields. {pull}28596[28596] {issue}28215[28215]
 - Remove deprecated `--template` and `--ilm-policy` flags. Use `--index-management` instead. {pull}28870[28870]
 - Remove options `logging.files.suffix` and default to datetime endings. {pull}28927[28927]
+- Remove Journalbeat. Use `journald` input of Filebeat instead. {pull}29131[29131]
 
 *Auditbeat*
 
@@ -69,11 +70,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Heartbeat*
 - Change behavior in case of duplicate monitor IDs in configs to be last monitor wins. {pull}29041[29041]
 
-*Journalbeat*
-
-- Rename field `journald.process.capabilites` to `journald.process.capabilities` to fix spelling. {pull}28065[28065]
-- Rename field `log.syslog.facility.name` to `log.syslog.facility.code` because the value is numeric rather than the facility name. {pull}28065[28065]
-
 *Metricbeat*
 
 - Add Linux pressure metricset {pull}27355[27355]
@@ -145,6 +141,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `fingerprint` processor to give it access to the `@timestamp` field. {issue}28683[28683]
 - Fix the wrong beat name on monitoring and state endpoint {issue}27755[27755]
 - Skip configuration checks in autodiscover for configurations that are already running {pull}29048[29048]
+- Fix `decode_json_processor` to always respect `add_error_key` {pull}29107[29107]
+- Fix `add_labels` flattening of array values. {pull}29211[29211]
 
 *Auditbeat*
 
@@ -191,6 +189,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Revert usageDetails api version to 2019-01-01. {pull}28995[28995]
 - Fix in `aws-s3` input regarding provider discovery through endpoint {pull}28963[28963]
 - Fix `threatintel.misp` filters configuration. {issue}27970[27970]
+- Fix opening files on Windows in filestream so open files can be deleted. {issue}29113[29113] {pull}29180[29180]
 
 *Heartbeat*
 
@@ -201,9 +200,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove accidentally included cups library in docker images. {pull}28853[pull]
 - Fix broken monitors with newer versions of image relying on dup3. {pull}28938[pull]
 
-*Journalbeat*
-
-
 *Metricbeat*
 
 - Fix checking tagsFilter using length in cloudwatch metricset. {pull}14525[14525]
@@ -241,6 +237,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Packetbeat*
 
+- Prevent incorrect use of AMQP protocol parsing from causing silent failure. {pull}29017[29017]
+- Fix error handling in MongoDB protocol parsing. {pull}29017[29017]
 
 *Winlogbeat*
 
@@ -348,15 +346,14 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support in aws-s3 input for s3 notification from SNS to SQS. {pull}28800[28800]
 - Add support in aws-s3 input for custom script parsing of s3 notifications. {pull}28946[28946]
 - Improve error handling in aws-s3 input for malformed s3 notifications. {issue}28828[28828] {pull}28946[28946]
+- Add support for parsers on journald input {pull}29070[29070]
 
 *Heartbeat*
 
 - Support JSON expressions / validation of JSON arrays. {pull}28073[28073]
 - Experimental 'run once' mode. {pull}25972[25972]
 - Add `keyword` multi-field mapping for `synthetics.step.name`. {pull}28452[28452]
 
-*Journalbeat*
-
 *Metricbeat*
 
 - Move the windows pdh implementation from perfmon to a shared location in order for future modules/metricsets to make use of. {pull}15503[15503]
@@ -401,8 +398,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Heartbeat*
 
-*Journalbeat*
-
 *Metricbeat*
 
 

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -457,8 +457,6 @@ def pushCIDockerImages(Map args = [:]) {
       tagAndPush(beatName: 'filebeat', arch: arch)
     } else if (beatsFolder.endsWith('heartbeat')) {
       tagAndPush(beatName: 'heartbeat', arch: arch)
-    } else if ("${beatsFolder}" == "journalbeat"){
-      tagAndPush(beatName: 'journalbeat', arch: arch)
     } else if (beatsFolder.endsWith('metricbeat')) {
       tagAndPush(beatName: 'metricbeat', arch: arch)
     } else if ("${beatsFolder}" == "packetbeat"){

diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 BUILD_DIR=$(CURDIR)/build
 COVERAGE_DIR=$(BUILD_DIR)/coverage
-BEATS?=auditbeat filebeat heartbeat journalbeat metricbeat packetbeat winlogbeat x-pack/functionbeat x-pack/elastic-agent x-pack/osquerybeat
+BEATS?=auditbeat filebeat heartbeat metricbeat packetbeat winlogbeat x-pack/functionbeat x-pack/elastic-agent x-pack/osquerybeat
 PROJECTS=libbeat $(BEATS)
 PROJECTS_ENV=libbeat filebeat metricbeat
 PYTHON_ENV?=$(BUILD_DIR)/python-env

diff --git a/README.md b/README.md
@@ -23,7 +23,6 @@ Beat  | Description
 [Filebeat](https://github.com/elastic/beats/tree/master/filebeat) | Tails and ships log files
 [Functionbeat](https://github.com/elastic/beats/tree/master/x-pack/functionbeat) | Read and ships events from serverless infrastructure.
 [Heartbeat](https://github.com/elastic/beats/tree/master/heartbeat) | Ping remote services for availability
-[Journalbeat](https://github.com/elastic/beats/tree/master/journalbeat) | Read and ships event from Journald.
 [Metricbeat](https://github.com/elastic/beats/tree/master/metricbeat) | Fetches sets of metrics from the operating system and services
 [Packetbeat](https://github.com/elastic/beats/tree/master/packetbeat) | Monitors the network and applications by sniffing packets
 [Winlogbeat](https://github.com/elastic/beats/tree/master/winlogbeat) | Fetches and ships Windows Event logs
@@ -45,7 +44,6 @@ on the [elastic.co site](https://www.elastic.co/guide/):
 * [Filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/index.html)
 * [Functionbeat](https://www.elastic.co/guide/en/beats/functionbeat/current/index.html)
 * [Heartbeat](https://www.elastic.co/guide/en/beats/heartbeat/current/index.html)
-* [Journalbeat](https://www.elastic.co/guide/en/beats/journalbeat/current/index.html)
 * [Metricbeat](https://www.elastic.co/guide/en/beats/metricbeat/current/index.html)
 * [Packetbeat](https://www.elastic.co/guide/en/beats/packetbeat/current/index.html)
 * [Winlogbeat](https://www.elastic.co/guide/en/beats/winlogbeat/current/index.html)

diff --git a/auditbeat/cmd/root.go b/auditbeat/cmd/root.go
@@ -63,7 +63,7 @@ func AuditbeatSettings() instance.Settings {
 	}
 }
 
-// Initialize initializes the entrypoint commands for journalbeat
+// Initialize initializes the entrypoint commands for auditbeat
 func Initialize(settings instance.Settings) *cmd.BeatsRootCmd {
 	create := beater.Creator(
 		beater.WithModuleOptions(

diff --git a/deploy/docker/journalbeat.docker.yml b/deploy/docker/journalbeat.docker.yml
diff --git a/dev-tools/cmd/update_go/update_go_version.go b/dev-tools/cmd/update_go/update_go_version.go
@@ -30,7 +30,6 @@ var files = []string{
 	"auditbeat/Dockerfile",
 	"filebeat/Dockerfile",
 	"heartbeat/Dockerfile",
-	"journalbeat/Dockerfile",
 	"libbeat/Dockerfile",
 	"libbeat/docs/version.asciidoc",
 	"metricbeat/Dockerfile",

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -1954,20 +1954,6 @@
   alias: true
   beat: heartbeat
 
-# Journalbeat
-- from: host.name
-  to: host.hostname
-  alias: false
-  beat: journalbeat
-  comment: This field should not be renamed as it would cause issue some Beats and Journalbeat does not have dashboards
-  # This field should not be renamed as it would cause issue some Beats and Journalbeat does not have dashboards
-  rename: false
-
-- from: read_timestamp
-  to: event.created
-  alias: true
-  beat: journalbeat
-
 ## Winlogbeat
 
 # Alias to ECS fields

diff --git a/docs/devguide/create-metricset.asciidoc b/docs/devguide/create-metricset.asciidoc
@@ -1,6 +1,8 @@
 [[creating-metricsets]]
 === Creating a Metricset
 
+include::generator-support-note.asciidoc[tag=metricset-generator]
+
 A metricset is the part of a Metricbeat module that fetches and structures the
 data from the remote service. Each module can have multiple metricsets. In this guide, you learn how to create your own metricset.
 

diff --git a/docs/devguide/generator-support-note.asciidoc b/docs/devguide/generator-support-note.asciidoc
@@ -0,0 +1,13 @@
+// tag::metricset-generator[]
+IMPORTANT: Elastic provides no warranty or support for the code used to generate
+metricsets. The generator is mainly offered as guidance for developers who want
+to create their own data shippers.
+
+// end::metricset-generator[]
+
+// tag::filebeat-generator[]
+IMPORTANT: Elastic provides no warranty or support for the code used to generate
+modules and filesets. The generator is mainly offered as guidance for developers
+who want to create their own data shippers.
+
+// end::filebeat-generator[]
diff --git a/docs/devguide/modules-dev-guide.asciidoc b/docs/devguide/modules-dev-guide.asciidoc
@@ -1,6 +1,8 @@
 [[filebeat-modules-devguide]]
 == Creating a New Filebeat Module
 
+include::generator-support-note.asciidoc[tag=filebeat-generator]
+
 This guide will walk you through creating a new Filebeat module.
 
 All Filebeat modules currently live in the main

diff --git a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl
@@ -559,3 +559,22 @@ filebeat.inputs:
   # Configure stream to filter to a specific stream: stdout, stderr or all (default)
   #stream: all
 
+#------------------------------ Journald input --------------------------------
+# Journald input is experimental.
+#- type: journald
+  #enabled: true
+  #id: service-foo
+
+  # You may wish to have separate inputs for each service. You can use
+  # include_matches to specify a list of filter expressions that are
+  # applied as a logical OR. You may specify filter
+  #include_matches:
+    #- _SYSTEMD_UNIT=foo.service
+
+  # Parsers are also supported, here is an example of the multiline
+  # parser.
+  #parsers:
+  #- multiline:
+    #type: count
+    #count_lines: 3
+
diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -966,6 +966,25 @@ filebeat.inputs:
   # Configure stream to filter to a specific stream: stdout, stderr or all (default)
   #stream: all
 
+#------------------------------ Journald input --------------------------------
+# Journald input is experimental.
+#- type: journald
+  #enabled: true
+  #id: service-foo
+
+  # You may wish to have separate inputs for each service. You can use
+  # include_matches to specify a list of filter expressions that are
+  # applied as a logical OR. You may specify filter
+  #include_matches:
+    #- _SYSTEMD_UNIT=foo.service
+
+  # Parsers are also supported, here is an example of the multiline
+  # parser.
+  #parsers:
+  #- multiline:
+    #type: count
+    #count_lines: 3
+
 
 # =========================== Filebeat autodiscover ============================
 

diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go
@@ -29,6 +29,7 @@ import (
 	input "github.com/elastic/beats/v7/filebeat/input/v2"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/cleanup"
+	"github.com/elastic/beats/v7/libbeat/common/file"
 	"github.com/elastic/beats/v7/libbeat/common/match"
 	"github.com/elastic/beats/v7/libbeat/feature"
 	"github.com/elastic/beats/v7/libbeat/logp"
@@ -244,7 +245,7 @@ func (inp *filestream) openFile(log *logp.Logger, path string, offset int64) (*o
 	}
 
 	ok := false
-	f, err := os.OpenFile(path, os.O_RDONLY, os.FileMode(0))
+	f, err := file.ReadOpen(path)
 	if err != nil {
 		return nil, fmt.Errorf("failed opening %s: %s", path, err)
 	}

diff --git a/filebeat/input/journald/config.go b/filebeat/input/journald/config.go
@@ -24,8 +24,9 @@ import (
 	"errors"
 	"time"
 
-	"github.com/elastic/beats/v7/journalbeat/pkg/journalfield"
-	"github.com/elastic/beats/v7/journalbeat/pkg/journalread"
+	"github.com/elastic/beats/v7/filebeat/input/journald/pkg/journalfield"
+	"github.com/elastic/beats/v7/filebeat/input/journald/pkg/journalread"
+	"github.com/elastic/beats/v7/libbeat/reader/parser"
 )
 
 // Config stores the options of a journald input.
@@ -51,6 +52,9 @@ type config struct {
 
 	// SaveRemoteHostname defines if the original source of the entry needs to be saved.
 	SaveRemoteHostname bool `config:"save_remote_hostname"`
+
+	// Parsers configuration
+	Parsers parser.Config `config:",inline"`
 }
 
 var errInvalidSeekFallback = errors.New("invalid setting for cursor_seek_fallback")

diff --git a/filebeat/input/journald/conv.go b/filebeat/input/journald/conv.go
@@ -23,7 +23,7 @@ package journald
 import (
 	"time"
 
-	"github.com/elastic/beats/v7/journalbeat/pkg/journalfield"
+	"github.com/elastic/beats/v7/filebeat/input/journald/pkg/journalfield"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/logp"
 )