Replace github.com/coreos/go-systemd/v22/sdjournal by journalctl

CHANGELOG-developer.next.asciidoc

@@ -69,6 +69,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Beats publishing pipeline does not propagate the close signal to its clients any more. It's responsibility of the user to close the pipeline client. {issue}38197[38197] {pull}38556[38556]
 - Debug log entries from the acker (`stateful ack ...` or `stateless ack ...`) removed. {pull}39672[39672]
 - Rename x-pack/filebeat websocket input to streaming. {issue}40264[40264] {pull}40421[40421]
+- Journald input now calls `journalctl` instead of using `github.com/coreos/go-systemd/[email protected]/sdjournal`, the CGO dependency has been removed from Filebeat {pull}40061[40061]
 
 ==== Bugfixes
 

CHANGELOG.next.asciidoc

@@ -13,6 +13,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Update Go version to 1.22.5. {pull}40082[40082]
 - Fix FQDN being lowercased when used as `host.hostname` {issue}39993[39993]
 - Beats won't log start up information when running under the Elastic Agent {40390}40390[40390]
+- Filebeat now needs `dup3`, `faccessat2`, `prctl` and `setrlimit` syscalls to run the journald input. If this input is not being used, the syscalls are not needed. All Beats have those syscalls allowed now because the default seccomp policy is global to all Beats. {pull}40061[40061]
 
 *Auditbeat*
 
@@ -53,6 +54,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Implement Elastic Agent status and health reporting for Winlog Filebeat input. {pull}40163[40163]
 - Fix filestream's registry GC: registry entries will never be removed if clean_inactive is set to "-1". {pull}40258[40258]
 - Added `ignore_empty_values` flag in `decode_cef` Filebeat processor. {pull}40268[40268]
+- Journald: removed configuration options `include_matches.or`, `include_matches.and`, `backoff`, `max_backoff`, `cursor_seek_fallback`. {pull}40061[40061]
+- Journald: `include_matches.match` now behaves in the same way as matchers in `journalctl`. Users should carefully update their input configuration. {pull}40061[40061]
+- Journald: `seek` and `since` behaviour have been simplified, if there is a cursor (state) `seek` and `since` are ignored and the cursor is used. {pull}40061[40061]
 
 *Heartbeat*
 
@@ -166,6 +170,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix bug in CEL input rate limit logic. {issue}40106[40106] {pull}40270[40270]
 - Relax requirements in Okta entity analytics provider user and device profile data shape. {pull}40359[40359]
 - Fix bug in Okta entity analytics rate limit logic. {issue}40106[40106] {pull}40267[40267]
+- Fix crashes in the journald input. {pull}40061[40061]
 
 *Heartbeat*
 

Vagrantfile

@@ -47,10 +47,13 @@ TEST_BOXES = [
   {:name => "ubuntu1604", :box => "ubuntu/xenial64", :platform => "ubuntu"},
   {:name => "ubuntu1804", :box => "ubuntu/bionic64", :platform => "ubuntu"},
   {:name => "ubuntu2004", :box => "ubuntu/focal64", :platform => "ubuntu"},
+  {:name => "ubuntu2204", :box => "ubuntu/jammy64", :platform => "ubuntu"},
 
   {:name => "debian8", :box => "generic/debian8", :platform => "debian"},
   {:name => "debian9", :box => "debian/stretch64", :platform => "debian"},
   {:name => "debian10", :box => "debian/buster64", :platform => "debian"},
+  {:name => "debian11", :box => "debian/bullseye64", :platform => "debian"},
+  {:name => "debian12", :box => "debian/bookworm64", :platform => "debian"},
 
   {:name => "amazon1", :box => "mvbcoding/awslinux", :platform => "centos"},
   {:name => "amazon2", :box => "bento/amazonlinux-2", :platform => "centos"},

filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl

@@ -775,9 +775,8 @@ filebeat.inputs:
   # You may wish to have separate inputs for each service. You can use
   # include_matches.or to specify a list of filter expressions that are
   # applied as a logical OR. You may specify filter
-  #include_matches.or:
-    #- equals:
-      #- _SYSTEMD_UNIT=foo.service
+  #include_matches.match:
+    #- _SYSTEMD_UNIT=foo.service
 
   # List of syslog identifiers
   #syslog_identifiers: ["audit"]

filebeat/docs/inputs/input-journald.asciidoc

@@ -11,7 +11,13 @@ experimental[]
 
 https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html[`journald`]
 is a system service that collects and stores logging data. The `journald` input
-reads this log data and the metadata associated with it.
+reads this log data and the metadata associated with it. To read this
+log data {beatname_uc} calls `journalctl` to read from the journal, therefore
+{beatname_uc} needs permission to execute `journalctl`.
+
+If the `journalctl` process exits unexpectedly the {type} input will
+terminate with an error and {beatname_uc} will need to be
+restarted to start reading from the jouranl again.
 
 The simplest configuration example is one that reads all logs from the default
 journal.
@@ -71,21 +77,22 @@ The `journald` input supports the following configuration options plus the
 [id="{beatname_lc}-input-{type}-id"]
 ==== `id`
 
-An optional unique identifier for the input. By providing a unique `id` you can
+An unique identifier for the input. By providing a unique `id` you can
 operate multiple inputs on the same journal. This allows each input's cursor to
-be persisted independently in the registry file.
+be persisted independently in the registry file. Each {type} input must have
+an unique ID.
 
 ["source","yaml",subs="attributes"]
 ----
 {beatname_lc}.inputs:
 - type: journald
   id: consul.service
-  include_matches:
+  include_matches.match:
     - _SYSTEMD_UNIT=consul.service
 
 - type: journald
   id: vault.service
-  include_matches:
+  include_matches.match:
     - _SYSTEMD_UNIT=vault.service
 ----
 
@@ -100,20 +107,6 @@ into a single journal and reads them.
 
 If no paths are specified, {beatname_uc} reads from the default journal.
 
-[float]
-[id="{beatname_lc}-input-{type}-backoff"]
-==== `backoff`
-
-The number of seconds to wait before trying to read again from journals. The
-default is 1s.
-
-[float]
-[id="{beatname_lc}-input-{type}-max-backoff"]
-==== `max_backoff`
-
-The maximum number of seconds to wait before attempting to read again from
-journals. The default is 60s.
-
 [float]
 [id="{beatname_lc}-input-{type}-seek"]
 ==== `seek`
@@ -124,36 +117,26 @@ The position to start reading the journal from. Valid settings are:
 {beatname_uc} resends all log messages in the journal.
 * `tail`: Starts reading at the end of the journal. This means that no events
 will be sent until a new message is written.
-* `cursor`: On first read, starts reading at the beginning of the journal. After
-a reload or restart, continues reading at the last known position.
 * `since`: Use the `since` option to determine where to start reading from.
 
-If you have old log files and want to skip lines, start {beatname_uc} with
-`seek: tail` specified. Then stop {beatname_uc}, set `seek: cursor`, and restart
-{beatname_uc}.
-
-[float]
-[id="{beatname_lc}-input-{type}-cursor_seek_fallback"]
-==== `cursor_seek_fallback`
+Regardless of the value of `seek` if {beatname_uc} has a state (cursor) for this
+input, the `seek` value is ignored and the current cursor is used. To reset
+the cursor, just change the `id` of the input, this will start from a fresh state.
 
-The position to start reading the journal from if no cursor information is
-available. Valid options are `head`, `tail` and `since`.
 
 [float]
 [id="{beatname_lc}-input-{type}-since"]
 ==== `since`
 
 A time offset from the current time to start reading from. To use
-`since`, either the `seek` option must be set to `since`, or the `seek` mode
-must be set to `cursor` and the `cursor_seek_fallback` set to `since`.
+`since`, `seek` option must be set to `since`.
 
 This example demonstrates how to resume from the persisted cursor when
 it exists, or otherwise begin reading logs from the last 24 hours.
 
 ["source","yaml",subs="attributes"]
 ----
-seek: cursor
-cursor_seek_fallback: since
+seek: since
 since: -24h
 ----
 
@@ -199,20 +182,30 @@ If the filter expressions apply to different fields, only entries with all field
 If they apply to the same fields, only entries where the field takes one of the specified values will be iterated.
 
 `match`: List of filter expressions to match fields.
-`or`: The filter expressions listed under `or` are connected with a disjunction (or).
-`and`: The filter expressions listed under `and` are connected with a conjunction (and).
 
 Please note that these expressions are limited. You can build complex filtering, but full logical
 expressions are not supported.
 
-The following include matches configuration reads all `systemd` syslog entries:
+The following include matches configuration will ingest entries that
+contain `journald.process.name: systemd` and `systemd.transport: syslog`.
+
+["source","yaml",subs="attributes"]
+----
+include_matches:
+  match:
+    - "journald.process.name=systemd"
+    - "systemd.transport=syslog"
+----
+
+The following include matches configuration will ingest entries that
+contain `systemd.transport: systemd` or `systemd.transport: kernel`.
 
 ["source","yaml",subs="attributes"]
 ----
-include_matches.and:
-- match:
-  - "journald.process.name=systemd"
-  - "systemd.transport=syslog"
+include_matches:
+  match:
+    - "systemd.transport=kernel"
+    - "systemd.transport=syslog"
 ----
 
 To reference fields, use one of the following:

filebeat/filebeat.reference.yml

@@ -1182,9 +1182,8 @@ filebeat.inputs:
   # You may wish to have separate inputs for each service. You can use
   # include_matches.or to specify a list of filter expressions that are
   # applied as a logical OR. You may specify filter
-  #include_matches.or:
-    #- equals:
-      #- _SYSTEMD_UNIT=foo.service
+  #include_matches.match:
+    #- _SYSTEMD_UNIT=foo.service
 
   # List of syslog identifiers
   #syslog_identifiers: ["audit"]

filebeat/input/journald/config.go

@@ -15,19 +15,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//go:build linux && cgo && withjournald
+//go:build linux
 
 package journald
 
 import (
-	"errors"
 	"sync"
 	"time"
 
 	"github.com/elastic/go-ucfg"
 
+	"github.com/elastic/beats/v7/filebeat/input/journald/pkg/journalctl"
 	"github.com/elastic/beats/v7/filebeat/input/journald/pkg/journalfield"
-	"github.com/elastic/beats/v7/filebeat/input/journald/pkg/journalread"
+
 	"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/v7/libbeat/reader/parser"
 )
@@ -41,22 +41,12 @@ type config struct {
 	// Paths stores the paths to the journal files to be read.
 	Paths []string `config:"paths"`
 
-	// Backoff is the current interval to wait before
-	// attempting to read again from the journal.
-	Backoff time.Duration `config:"backoff" validate:"min=0,nonzero"`
-
-	// MaxBackoff is the limit of the backoff time.
-	MaxBackoff time.Duration `config:"max_backoff" validate:"min=0,nonzero"`
-
 	// Since is the relative time offset from now to provide journal
-	// entries from. If Since is nil, no offset is applied.
-	Since *time.Duration `config:"since"`
+	// entries from.
+	Since time.Duration `config:"since"`
 
 	// Seek is the method to read from journals.
-	Seek journalread.SeekMode `config:"seek"`
-
-	// CursorSeekFallback sets where to seek if registry file is not available.
-	CursorSeekFallback journalread.SeekMode `config:"cursor_seek_fallback"`
+	Seek journalctl.SeekMode `config:"seek"`
 
 	// Matches store the key value pairs to match entries.
 	Matches bwcIncludeMatches `config:"include_matches"`
@@ -89,11 +79,8 @@ func (im *bwcIncludeMatches) Unpack(c *ucfg.Config) error {
 		if err := c.Unpack(&matches); err != nil {
 			return err
 		}
-		for _, x := range matches {
-			im.OR = append(im.OR, journalfield.IncludeMatches{
-				Matches: []journalfield.Matcher{x},
-			})
-		}
+		im.Matches = append(im.Matches, matches...)
+
 		includeMatchesWarnOnce.Do(func() {
 			cfgwarn.Deprecate("", "Please migrate your journald input's "+
 				"include_matches config to the new more expressive format.")
@@ -104,43 +91,9 @@ func (im *bwcIncludeMatches) Unpack(c *ucfg.Config) error {
 	return c.Unpack((*journalfield.IncludeMatches)(im))
 }
 
-var (
-	errInvalidSeekFallback = errors.New("invalid setting for cursor_seek_fallback")
-	errInvalidSeek         = errors.New("invalid setting for seek")
-	errInvalidSeekSince    = errors.New("incompatible setting for since and seek or cursor_seek_fallback")
-)
-
 func defaultConfig() config {
 	return config{
-		Backoff:            1 * time.Second,
-		MaxBackoff:         20 * time.Second,
-		Seek:               journalread.SeekCursor,
-		CursorSeekFallback: journalread.SeekHead,
+		Seek:               journalctl.SeekHead,
 		SaveRemoteHostname: false,
 	}
 }
-
-func (c *config) Validate() error {
-	if c.Seek == journalread.SeekInvalid {
-		return errInvalidSeek
-	}
-	switch c.CursorSeekFallback {
-	case journalread.SeekHead, journalread.SeekTail, journalread.SeekSince:
-	default:
-		return errInvalidSeekFallback
-	}
-	if c.Since == nil {
-		switch {
-		case c.Seek == journalread.SeekSince,
-			c.Seek == journalread.SeekCursor && c.CursorSeekFallback == journalread.SeekSince:
-			return errInvalidSeekSince
-		default:
-			return nil
-		}
-	}
-	needSince := c.Seek == journalread.SeekSince || (c.Seek == journalread.SeekCursor && c.CursorSeekFallback == journalread.SeekSince)
-	if !needSince {
-		return errInvalidSeekSince
-	}
-	return nil
-}