[auditbeat] fim: implement ebpf backend (elastic#37223)

* fim: implement ebpf backend

CHANGELOG.next.asciidoc

@@ -134,6 +134,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d
 *Auditbeat*
 
 - Add linux capabilities to processes in the system/process. {pull}37453[37453]
+- Add opt-in eBPF backend for file_integrity module. {pull}37223[37223]
 
 *Filebeat*
 

NOTICE.txt

@@ -12257,11 +12257,11 @@ SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/ebpfevents
-Version: v0.3.2
+Version: v0.4.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.3.2/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.4.0/LICENSE.txt:
 
 The https://github.com/elastic/ebpfevents repository contains source code under
 various licenses:

auditbeat/.gitignore

@@ -6,4 +6,3 @@ module/*/_meta/config.yml
 /auditbeat
 /auditbeat.test
 /docs/html_docs
-

auditbeat/auditbeat.reference.yml

@@ -92,6 +92,11 @@ auditbeat.modules:
   # Auditbeat will ignore files unless they match a pattern.
   #include_files:
   #- '/\.ssh($|/)'
+  # Select the backend which will be used to source events.
+  # "fsnotify" doesn't have the ability to associate user data to file events.
+  # Valid values: auto, fsnotify, kprobes, ebpf.
+  # Default: fsnotify.
+  backend: fsnotify
 
   # Scan over the configured file paths at startup and send events for new or
   # modified files since the last time Auditbeat was running.

auditbeat/docker-compose.yml

@@ -14,11 +14,15 @@ services:
       - KIBANA_PORT=5601
     volumes:
       - ${PWD}/..:/go/src/github.com/elastic/beats/
+      - /sys:/sys
     command: make
     privileged: true
     pid: host
     cap_add:
       - AUDIT_CONTROL
+      - BPF
+      - PERFMON
+      - SYS_RESOURCE
 
   # This is a proxy used to block beats until all services are healthy.
   # See: https://github.com/docker/compose/issues/4369

auditbeat/docs/modules/file_integrity.asciidoc

@@ -28,8 +28,13 @@ to only send events for new or modified files.
 
 The operating system features that power this feature are as follows.
 
-* Linux - `inotify` is used, and therefore the kernel must have inotify support.
+* Linux - Multiple backends are supported: `auto`, `fsnotify`, `kprobes`, `ebpf`.
+By default, `fsnotify` is used, and therefore the kernel must have inotify support.
 Inotify was initially merged into the 2.6.13 Linux kernel.
+The eBPF backend uses modern eBPF features and supports 5.10.16+ kernels.
+FSNotify doesn't have the ability to associate user data to file events.
+The preferred backend can be selected by specifying the `backend` config option.
+Since eBPF and Kprobes are in technical preview, `auto` will default to `fsnotify`.
 * macOS (Darwin) - Uses the `FSEvents` API, present since macOS 10.5. This API
 coalesces multiple changes to a file into a single event. {beatname_uc} translates
 this coalesced changes into a meaningful sequence of actions. However,
@@ -144,6 +149,9 @@ of this directories are watched. If `recursive` is set to `true`, the
 `file_integrity` module will watch for changes on this directories and all
 their subdirectories.
 
+*`backend`*:: (*Linux only*) Select the backend which will be used to
+source events. Valid values: `auto`, `fsnotify`, `kprobes`, `ebpf`. Default: `fsnotify`.
+
 include::{docdir}/auditbeat-options.asciidoc[]
 
 

auditbeat/module/file_integrity/_meta/config.yml.tmpl

@@ -55,6 +55,14 @@
   #- '/\.ssh($|/)'
   {{- end }}
 
+  {{- if eq .GOOS "linux" }}
+  # Select the backend which will be used to source events.
+  # "fsnotify" doesn't have the ability to associate user data to file events.
+  # Valid values: auto, fsnotify, kprobes, ebpf.
+  # Default: fsnotify.
+  backend: fsnotify
+  {{- end }}
+
   # Scan over the configured file paths at startup and send events for new or
   # modified files since the last time Auditbeat was running.
   scan_at_start: true

auditbeat/module/file_integrity/_meta/docs.asciidoc

@@ -21,8 +21,13 @@ to only send events for new or modified files.
 
 The operating system features that power this feature are as follows.
 
-* Linux - `inotify` is used, and therefore the kernel must have inotify support.
+* Linux - Multiple backends are supported: `auto`, `fsnotify`, `kprobes`, `ebpf`.
+By default, `fsnotify` is used, and therefore the kernel must have inotify support.
 Inotify was initially merged into the 2.6.13 Linux kernel.
+The eBPF backend uses modern eBPF features and supports 5.10.16+ kernels.
+FSNotify doesn't have the ability to associate user data to file events.
+The preferred backend can be selected by specifying the `backend` config option.
+Since eBPF and Kprobes are in technical preview, `auto` will default to `fsnotify`.
 * macOS (Darwin) - Uses the `FSEvents` API, present since macOS 10.5. This API
 coalesces multiple changes to a file into a single event. {beatname_uc} translates
 this coalesced changes into a meaningful sequence of actions. However,
@@ -137,4 +142,7 @@ of this directories are watched. If `recursive` is set to `true`, the
 `file_integrity` module will watch for changes on this directories and all
 their subdirectories.
 
+*`backend`*:: (*Linux only*) Select the backend which will be used to
+source events. Valid values: `auto`, `fsnotify`, `kprobes`, `ebpf`. Default: `fsnotify`.
+
 include::{docdir}/auditbeat-options.asciidoc[]

auditbeat/module/file_integrity/config.go

@@ -18,10 +18,12 @@
 package file_integrity
 
 import (
+	"errors"
 	"fmt"
 	"math"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"sort"
 	"strings"
 
@@ -72,6 +74,25 @@ const (
 	XXH64       HashType = "xxh64"
 )
 
+type Backend string
+
+const (
+	BackendFSNotify Backend = "fsnotify"
+	BackendKprobes  Backend = "kprobes"
+	BackendEBPF     Backend = "ebpf"
+	BackendAuto     Backend = "auto"
+)
+
+func (b *Backend) Unpack(v string) error {
+	*b = Backend(v)
+	switch *b {
+	case BackendFSNotify, BackendKprobes, BackendEBPF, BackendAuto:
+		return nil
+	default:
+		return fmt.Errorf("invalid backend: %q", v)
+	}
+}
+
 // Config contains the configuration parameters for the file integrity
 // metricset.
 type Config struct {
@@ -86,6 +107,7 @@ type Config struct {
 	Recursive           bool            `config:"recursive"` // Recursive enables recursive monitoring of directories.
 	ExcludeFiles        []match.Matcher `config:"exclude_files"`
 	IncludeFiles        []match.Matcher `config:"include_files"`
+	Backend             Backend         `config:"backend"`
 }
 
 // Validate validates the config data and return an error explaining all the
@@ -160,6 +182,11 @@ nextHash:
 	if err != nil {
 		errs = append(errs, fmt.Errorf("invalid scan_rate_per_sec value: %w", err))
 	}
+
+	if c.Backend != "" && c.Backend != BackendAuto && runtime.GOOS != "linux" {
+		errs = append(errs, errors.New("backend can only be specified on linux"))
+	}
+
 	return errs.Err()
 }
 

auditbeat/module/file_integrity/event.go

@@ -65,11 +65,14 @@ const (
 	// SourceFSNotify identifies events triggered by a notification from the
 	// file system.
 	SourceFSNotify
+	// SourceEBPF identifies events triggered by an eBPF program.
+	SourceEBPF
 )
 
 var sourceNames = map[Source]string{
 	SourceScan:     "scan",
 	SourceFSNotify: "fsnotify",
+	SourceEBPF:     "ebpf",
 }
 
 // Type identifies the file type (e.g. dir, file, symlink).
@@ -91,12 +94,20 @@ const (
 	FileType
 	DirType
 	SymlinkType
+	CharDeviceType
+	BlockDeviceType
+	FIFOType
+	SocketType
 )
 
 var typeNames = map[Type]string{
-	FileType:    "file",
-	DirType:     "dir",
-	SymlinkType: "symlink",
+	FileType:        "file",
+	DirType:         "dir",
+	SymlinkType:     "symlink",
+	CharDeviceType:  "char_device",
+	BlockDeviceType: "block_device",
+	FIFOType:        "fifo",
+	SocketType:      "socket",
 }
 
 // Digest is an output of a hash function.
@@ -189,36 +200,42 @@ func NewEventFromFileInfo(
 
 	switch event.Info.Type {
 	case FileType:
-		if event.Info.Size <= maxFileSize {
-			hashes, nbytes, err := hashFile(event.Path, maxFileSize, hashTypes...)
-			if err != nil {
-				event.errors = append(event.errors, err)
-				event.hashFailed = true
-			} else if hashes != nil {
-				// hashFile returns nil hashes and no error when:
-				// - There's no hashes configured.
-				// - File size at the time of hashing is larger than configured limit.
-				event.Hashes = hashes
-				event.Info.Size = nbytes
-			}
-
-			if len(fileParsers) != 0 && event.ParserResults == nil {
-				event.ParserResults = make(mapstr.M)
-			}
-			for _, p := range fileParsers {
-				err = p.Parse(event.ParserResults, path)
-				if err != nil {
-					event.errors = append(event.errors, err)
-				}
-			}
-		}
+		fillHashes(&event, path, maxFileSize, hashTypes, fileParsers)
 	case SymlinkType:
-		event.TargetPath, _ = filepath.EvalSymlinks(event.Path)
+		event.TargetPath, err = filepath.EvalSymlinks(event.Path)
+		if err != nil {
+			event.errors = append(event.errors, err)
+		}
 	}
 
 	return event
 }
 
+func fillHashes(event *Event, path string, maxFileSize uint64, hashTypes []HashType, fileParsers []FileParser) {
+	if event.Info.Size <= maxFileSize {
+		hashes, nbytes, err := hashFile(event.Path, maxFileSize, hashTypes...)
+		if err != nil {
+			event.errors = append(event.errors, err)
+			event.hashFailed = true
+		} else if hashes != nil {
+			// hashFile returns nil hashes and no error when:
+			// - There's no hashes configured.
+			// - File size at the time of hashing is larger than configured limit.
+			event.Hashes = hashes
+			event.Info.Size = nbytes
+		}
+
+		if len(fileParsers) != 0 && event.ParserResults == nil {
+			event.ParserResults = make(mapstr.M)
+		}
+		for _, p := range fileParsers {
+			if err = p.Parse(event.ParserResults, path); err != nil {
+				event.errors = append(event.errors, err)
+			}
+		}
+	}
+}
+
 // NewEvent creates a new Event. Any errors that occur are included in the
 // returned Event.
 func NewEvent(