Auditbeat: file_integrity module tests failure

[oakrizan]

Flaky Test

auditbeat/module/file_integrity/fileinfo_test.go

• Test Name: TestNewMetadata
  Link:

  beats/auditbeat/module/file_integrity/fileinfo_test.go

  Lines 32 to 98 in 49f3efe

  	func TestNewMetadata(t *testing.T) {
  	// Can be removed after https://github.com/elastic/beats/issues/37701 is solved
  	skipOnBuildkiteDarwin(t, "Group check")
  	f, err := os.CreateTemp(t.TempDir(), "metadata")
  	if err != nil {
  	t.Fatal(err)
  	}
  	_, err = f.WriteString("metadata test")
  	if err != nil {
  	t.Fatal(err)
  	}
  	require.NoError(t, f.Sync())
  	f.Close()
  	info, err := os.Lstat(f.Name())
  	if err != nil {
  	t.Fatal(err)
  	}
  	meta, err := NewMetadata(f.Name(), info)
  	if err != nil {
  	t.Fatal(err)
  	}
  	u, err := user.Current()
  	if err != nil {
  	t.Fatal(err)
  	}
  	assert.NotZero(t, meta.Inode)
  	if runtime.GOOS == "windows" {
  	// The owner can differ from the creator if the GPO for
  	// "System object Default owner for objects created by members of the Administrators group"
  	// is set to "administrators group" rather than "object creator".
  	if meta.Owner == u.Username {
  	assert.Equal(t, u.Uid, meta.SID)
  	} else if meta.Owner == `BUILTIN\Administrators` {
  	// Well-known SID for BUILTIN_ADMINISTRATORS.
  	assert.Equal(t, "S-1-5-32-544", meta.SID)
  	} else {
  	t.Error("unexpected owner", meta.Owner)
  	}
  	assert.Zero(t, meta.UID)
  	assert.Zero(t, meta.GID)
  	assert.Empty(t, meta.Group)
  	} else {
  	group, err := user.LookupGroupId(u.Gid)
  	if err != nil {
  	t.Fatal(err)
  	}
  	assert.Equal(t, u.Uid, strconv.Itoa(int(meta.UID)))
  	assert.Equal(t, u.Gid, strconv.Itoa(int(meta.GID)))
  	assert.Equal(t, u.Username, meta.Owner)
  	assert.Equal(t, group.Name, meta.Group)
  	assert.Empty(t, meta.SID)
  	assert.EqualValues(t, 0o600, meta.Mode)
  	}
  	assert.EqualValues(t, len("metadata test"), meta.Size, "size")
  	assert.NotZero(t, meta.MTime, "mtime")
  	assert.NotZero(t, meta.CTime, "ctime")
  	assert.Equal(t, FileType, meta.Type, "type")
  	}

• Test Name: TestSetUIDSetGIDBits
  Link:

  beats/auditbeat/module/file_integrity/fileinfo_test.go

  Lines 100 to 156 in 49f3efe

  	func TestSetUIDSetGIDBits(t *testing.T) {
  	// Can be removed after https://github.com/elastic/beats/issues/37701 is solved
  	skipOnBuildkiteDarwin(t, "Wheel permission issue")
  	f, err := os.CreateTemp(t.TempDir(), "setuid")
  	if err != nil {
  	t.Fatal(err)
  	}
  	_, err = f.WriteString("metadata test")
  	if err != nil {
  	t.Fatal(err)
  	}
  	require.NoError(t, f.Sync())
  	f.Close()
  	info, err := os.Lstat(f.Name())
  	if err != nil {
  	t.Fatal(err)
  	}
  	meta, err := NewMetadata(f.Name(), info)
  	if err != nil {
  	t.Fatal(err)
  	}
  	assert.False(t, meta.SetUID)
  	assert.False(t, meta.SetGID)
  	if runtime.GOOS == "windows" {
  	t.Skip("No setuid/setgid bits on Windows")
  	}
  	for _, flags := range []os.FileMode{
  	0o600 | os.ModeSetuid,
  	0o600 | os.ModeSetgid,
  	0o600 | os.ModeSetuid | os.ModeSetuid,
  	} {
  	msg := fmt.Sprintf("checking flags %04o", flags)
  	if err = os.Chmod(f.Name(), flags); err != nil {
  	t.Fatal(err, msg)
  	}
  	info, err = os.Lstat(f.Name())
  	if err != nil {
  	t.Fatal(err, msg)
  	}
  	meta, err = NewMetadata(f.Name(), info)
  	if err != nil {
  	t.Fatal(err)
  	}
  	assert.Equal(t, flags&os.ModeSetuid != 0, meta.SetUID)
  	assert.Equal(t, flags&os.ModeSetgid != 0, meta.SetGID)
  	}
  	}

auditbeat/module/file_integrity/metricset_test.go

• Test Name: TestActions
  Link:

  beats/auditbeat/module/file_integrity/metricset_test.go

  Lines 64 to 159 in 49f3efe

  	func TestActions(t *testing.T) {
  	skipOnCIForDarwinAMD64(t)
  	// Can be removed after https://github.com/elastic/ingest-dev/issues/3016 is solved
  	skipOnBuildkiteWindows(t)
  	// Can be removed after https://github.com/elastic/ingest-dev/issues/3076 is solved
  	skipOnBuildkiteDarwinArm(t)
  	defer abtest.SetupDataDir(t)()
  	bucket, err := datastore.OpenBucket(bucketName)
  	if err != nil {
  	t.Fatal(err)
  	}
  	defer bucket.Close()
  	// First directory
  	dir := t.TempDir()
  	// Second directory (to be reported with "initial_scan")
  	newDir := t.TempDir()
  	createdFilepath := filepath.Join(dir, "created.txt")
  	updatedFilepath := filepath.Join(dir, "updated.txt")
  	deletedFilepath := filepath.Join(dir, "deleted.txt")
  	// Add first directory to db (so that files in it are not reported with "initial_scan")
  	e := &Event{
  	Timestamp: time.Now().UTC(),
  	Path: dir,
  	Action: InitialScan,
  	}
  	if err = store(bucket, e); err != nil {
  	t.Fatal(err)
  	}
  	// Add fake event for non-existing file to db to simulate when a file has been deleted
  	deletedFileEvent := &Event{
  	Timestamp: time.Now().UTC(),
  	Path: deletedFilepath,
  	Action: Created,
  	}
  	if err = store(bucket, deletedFileEvent); err != nil {
  	t.Fatal(err)
  	}
  	// Insert fake file event into db to simulate when a file has changed
  	digest := sha1.New().Sum([]byte("different string"))
  	updatedFileEvent := &Event{
  	Timestamp: time.Now().UTC(),
  	Path: updatedFilepath,
  	Action: Created,
  	Hashes: map[HashType]Digest{SHA1: digest},
  	}
  	if err = store(bucket, updatedFileEvent); err != nil {
  	t.Fatal(err)
  	}
  	// Create some files in first directory
  	require.NoError(t, os.WriteFile(createdFilepath, []byte("hello world"), 0o600))
  	require.NoError(t, os.WriteFile(updatedFilepath, []byte("hello world"), 0o600))
  	ms := mbtest.NewPushMetricSetV2WithRegistry(t, getConfig(dir, newDir), ab.Registry)
  	events := mbtest.RunPushMetricSetV2(10*time.Second, 5, ms)
  	assert.Len(t, events, 5)
  	for _, event := range events {
  	if event.Error != nil {
  	t.Fatalf("received error: %+v", event.Error)
  	}
  	actions, err := event.MetricSetFields.GetValue("event.action")
  	path, err2 := event.MetricSetFields.GetValue("file.path")
  	if assert.NoError(t, err) && assert.NoError(t, err2) {
  	// Note: Actions reported for a file or directory will be different
  	// depending on whether the scanner or the platform-dependent
  	// filesystem event listener reported it. The subset of actions we test
  	// for here should be consistent across all cases though.
  	switch path.(string) {
  	case newDir:
  	assert.Contains(t, actions, "initial_scan")
  	case dir:
  	assert.Contains(t, actions, "attributes_modified")
  	case deletedFilepath:
  	assert.Contains(t, actions, "deleted")
  	case createdFilepath:
  	assert.Contains(t, actions, "created")
  	case updatedFilepath:
  	assert.Contains(t, actions, "updated")
  	assert.Contains(t, actions, "attributes_modified")
  	default:
  	assert.Fail(t, "unexpected path", "path %v", path)
  	}
  	}
  	}
  	}

• Test Name: TestExcludedFiles
  Link:

  beats/auditbeat/module/file_integrity/metricset_test.go

  Lines 161 to 211 in 49f3efe

  	func TestExcludedFiles(t *testing.T) {
  	skipOnCIForDarwinAMD64(t)
  	// Can be removed after https://github.com/elastic/ingest-dev/issues/3016 is solved
  	skipOnBuildkiteWindows(t)
  	// Can be removed after https://github.com/elastic/ingest-dev/issues/3076 is solved
  	skipOnBuildkiteDarwinArm(t)
  	defer abtest.SetupDataDir(t)()
  	bucket, err := datastore.OpenBucket(bucketName)
  	if err != nil {
  	t.Fatal(err)
  	}
  	defer bucket.Close()
  	dir := t.TempDir()
  	ms := mbtest.NewPushMetricSetV2WithRegistry(t, getConfig(dir), ab.Registry)
  	go func() {
  	for _, f := range []string{"FILE.TXT", "FILE.TXT.SWP", "file.txt.swo", ".git/HEAD", ".gitignore"} {
  	file := filepath.Join(dir, f)
  	_ = os.WriteFile(file, []byte("hello world"), 0o600)
  	}
  	}()
  	events := mbtest.RunPushMetricSetV2(10*time.Second, 3, ms)
  	for _, e := range events {
  	if e.Error != nil {
  	t.Fatalf("received error: %+v", e.Error)
  	}
  	}
  	wanted := map[string]bool{
  	dir: true,
  	filepath.Join(dir, "FILE.TXT"): true,
  	filepath.Join(dir, ".gitignore"): true,
  	}
  	if !assert.Len(t, events, len(wanted)) {
  	return
  	}
  	for _, e := range events {
  	event := e.MetricSetFields
  	path, err := event.GetValue("file.path")
  	if assert.NoError(t, err) {
  	_, ok := wanted[path.(string)]
  	assert.True(t, ok)
  	}
  	}
  	}

• Test Name: TestIncludedExcludedFiles
  Link:

  beats/auditbeat/module/file_integrity/metricset_test.go

  Lines 213 to 271 in 49f3efe

  	func TestIncludedExcludedFiles(t *testing.T) {
  	skipOnCIForDarwinAMD64(t)
  	// Can be removed after https://github.com/elastic/ingest-dev/issues/3016 is solved
  	skipOnBuildkiteWindows(t)
  	// Can be removed after https://github.com/elastic/ingest-dev/issues/3076 is solved
  	skipOnBuildkiteDarwinArm(t)
  	defer abtest.SetupDataDir(t)()
  	bucket, err := datastore.OpenBucket(bucketName)
  	if err != nil {
  	t.Fatal(err)
  	}
  	defer bucket.Close()
  	dir := t.TempDir()
  	err = os.Mkdir(filepath.Join(dir, ".ssh"), 0o700)
  	if err != nil {
  	t.Fatal(err)
  	}
  	config := getConfig(dir)
  	config["include_files"] = []string{`\.ssh`}
  	config["recursive"] = true
  	ms := mbtest.NewPushMetricSetV2WithRegistry(t, config, ab.Registry)
  	for _, f := range []string{"FILE.TXT", ".ssh/known_hosts", ".ssh/known_hosts.swp"} {
  	file := filepath.Join(dir, f)
  	require.NoError(t, os.WriteFile(file, []byte("hello world"), 0o600))
  	}
  	events := mbtest.RunPushMetricSetV2(10*time.Second, 3, ms)
  	for _, e := range events {
  	if e.Error != nil {
  	t.Fatalf("received error: %+v", e.Error)
  	}
  	}
  	wanted := map[string]bool{
  	dir: true,
  	filepath.Join(dir, ".ssh"): true,
  	filepath.Join(dir, ".ssh/known_hosts"): true,
  	}
  	if !assert.Len(t, events, len(wanted)) {
  	return
  	}
  	got := map[string]bool{}
  	for _, e := range events {
  	event := e.MetricSetFields
  	path, err := event.GetValue("file.path")
  	if assert.NoError(t, err, "Failed to read file.path field") {
  	got[path.(string)] = true
  	}
  	}
  	assert.Equal(t, wanted, got)
  	}

• Branch:
  -- Enable tests that were muted during migration #40387
  -- Enable tests that were muted during migration 7.17 #40398

NB! tests should be enabled back, since those were muted while migrating from Jenkins to Buildkite. Please check changed files in mentioned PRs:
-- auditbeat/module/file_integrity/fileinfo_test.go
-- auditbeat/module/file_integrity/metricset_test.go

• Artifact Link:
  macOS arm64: auditbeat_build_6789_mac-auditbeat-macos-arm64-unit-tests.log
  macOS x86_64: auditbeat_build_6789_mac-auditbeat-macos-x86-64-unit-tests.log
  win 2016 (other win failures are similar): auditbeat_build_6789_windows-auditbeat-win-2016-unit-tests.log

• Notes: Additional details about the test. e.g. theory as to failure cause

1. metricset_test.go failes on Windows due to shortened TMP, additional details: https://github.com/elastic/ingest-dev/issues/3016

2. Buildkite builds:
   -- macOS x86_64: https://buildkite.com/elastic/auditbeat/builds/6789#019102f4-24b0-478d-9549-970404e44521
   -- macOS arm64: https://buildkite.com/elastic/auditbeat/builds/6789#019102f4-24b2-43b3-ac36-04b9cdcdba09
   -- Win 2016: https://buildkite.com/elastic/auditbeat/builds/6789#019102f4-24a1-4cdb-9df0-10fb8a5c0151
   -- Win 2022: https://buildkite.com/elastic/auditbeat/builds/6789#019102f4-24a3-417f-9bd1-a0895f3d485a
   -- Win 2019: https://buildkite.com/elastic/auditbeat/builds/6789#019102f4-24b5-4e2b-8e80-cd65acbf9bea
   -- Win 10: https://buildkite.com/elastic/auditbeat/builds/6789#019102f4-24b7-4b04-8fd3-5e3e257cddf1
   -- Win 11: https://buildkite.com/elastic/auditbeat/builds/6789#019102f4-24b9-48ab-916b-f3e42892e0eb
   -- macOS x86_64 on 7.17 branch: https://buildkite.com/elastic/auditbeat/builds/6826#019107ea-0217-4614-bfd5-e8941bac565f/1893-1943

3. Tests failure per OS:

Beat	Test file	Test	OS
auditbeat	auditbeat/module/file_integrity/fileinfo_test.go	TestNewMetadata	macOS x86_64 macOS arm64
auditbeat	auditbeat/module/file_integrity/fileinfo_test.go	TestSetUIDSetGIDBits	macOS x86_64 macOS arm64
auditbeat	auditbeat/module/file_integrity/metricset_test.go	TestActions	macOS arm64 Windows (2016, 2019, 2022, 10, 11)
auditbeat	auditbeat/module/file_integrity/metricset_test.go	TestExcludedFiles	macOS arm64 Windows (2016, 2019, 2022, 10, 11)
auditbeat	auditbeat/module/file_integrity/metricset_test.go	TestIncludedExcludedFiles	macOS arm64 Windows (2016, 2019, 2022, 10, 11)

Stack Trace

=== FAIL: auditbeat/module/file_integrity TestNewMetadata (0.02s)
--
  | fileinfo_test.go:86:
  | Error Trace:	/Users/admin/builds/bk-agent-prod-orka-1722331758191225599/elastic/auditbeat/auditbeat/module/file_integrity/fileinfo_test.go:86
  | Error:      	Not equal:
  | expected: "20"
  | actual  : "0"
  |  
  | Diff:
  | --- Expected
  | +++ Actual
  | @@ -1 +1 @@
  | -20
  | +0
  | Test:       	TestNewMetadata
  | fileinfo_test.go:88:
  | Error Trace:	/Users/admin/builds/bk-agent-prod-orka-1722331758191225599/elastic/auditbeat/auditbeat/module/file_integrity/fileinfo_test.go:88
  | Error:      	Not equal:
  | expected: "staff"
  | actual  : "wheel"
  |  
  | Diff:
  | --- Expected
  | +++ Actual
  | @@ -1 +1 @@
  | -staff
  | +wheel
  | Test:       	TestNewMetadata
  |  
  | === FAIL: auditbeat/module/file_integrity TestSetUIDSetGIDBits (0.02s)
  | fileinfo_test.go:154:
  | Error Trace:	/Users/admin/builds/bk-agent-prod-orka-1722331758191225599/elastic/auditbeat/auditbeat/module/file_integrity/fileinfo_test.go:154
  | Error:      	Not equal:
  | expected: true
  | actual  : false
  | Test:       	TestSetUIDSetGIDBits
  |  
  | === FAIL: auditbeat/module/file_integrity TestActions (10.22s)
  | metricset_test.go:128:
  | Error Trace:	/Users/admin/builds/bk-agent-prod-orka-1722331758191225599/elastic/auditbeat/auditbeat/module/file_integrity/metricset_test.go:128
  | Error:      	"[{{} {} {"event":{"action":["initial_scan"],"category":["file"],"kind":"event","type":["info"]},"file":{"ctime":"2024-07-30T09:33:55.318592553Z","gid":"0","group":"wheel","inode":"520903","mode":"0755","mtime":"2024-07-30T09:33:55.318592553Z","owner":"admin","path":"/private/tmp/TestActions4021485366/001","type":"dir","uid":"501"}}    2024-07-30 09:33:55.347081 +0000 UTC <nil>   148.042µs 0s false} {{} {} {"event":{"action":["initial_scan"],"category":["file"],"kind":"event","type":["info"]},"file":{"ctime":"2024-07-30T09:33:55.31849897Z","extension":"txt","gid":"0","group":"wheel","hash":{"sha1":"2aae6c35c94fcfb415dbe95f408b9ce91ee846ed"},"inode":"520907","mode":"0600","mtime":"2024-07-30T09:33:55.31849897Z","owner":"admin","path":"/private/tmp/TestActions4021485366/001/created.txt","size":11,"type":"file","uid":"501"}}    2024-07-30 09:33:55.347173 +0000 UTC <nil>   102.875µs 0s false} {{} {} {"event":{"action":["initial_scan"],"category":["file"],"kind":"event","type":["info"]},"file":{"ctime":"2024-07-30T09:33:55.31863272Z","extension":"txt","gid":"0","group":"wheel","hash":{"sha1":"2aae6c35c94fcfb415dbe95f408b9ce91ee846ed"},"inode":"520908","mode":"0600","mtime":"2024-07-30T09:33:55.31863272Z","owner":"admin","path":"/private/tmp/TestActions4021485366/001/updated.txt","size":11,"type":"file","uid":"501"}}    2024-07-30 09:33:55.347275 +0000 UTC <nil>   76.458µs 0s false} {{} {} {"event":{"action":["initial_scan"],"category":["file"],"kind":"event","type":["info"]},"file":{"ctime":"2024-07-30T09:33:55.182588173Z","gid":"0","group":"wheel","inode":"520904","mode":"0755","mtime":"2024-07-30T09:33:55.182588173Z","owner":"admin","path":"/private/tmp/TestActions4021485366/002","type":"dir","uid":"501"}}    2024-07-30 09:33:55.373103 +0000 UTC <nil>   101.084µs 0s false}]" should have 5 item(s), but has 4
  | Test:       	TestActions
  | metricset_test.go:155:
  | Error Trace:	/Users/admin/builds/bk-agent-prod-orka-1722331758191225599/elastic/auditbeat/auditbeat/module/file_integrity/metricset_test.go:155
  | Error:      	unexpected path
  | Test:       	TestActions
  | Messages:   	path /private/tmp/TestActions4021485366/001
  | metricset_test.go:155:
  | Error Trace:	/Users/admin/builds/bk-agent-prod-orka-1722331758191225599/elastic/auditbeat/auditbeat/module/file_integrity/metricset_test.go:155
  | Error:      	unexpected path
  | Test:       	TestActions
  | Messages:   	path /private/tmp/TestActions4021485366/001/created.txt
  | metricset_test.go:155:
  | Error Trace:	/Users/admin/builds/bk-agent-prod-orka-1722331758191225599/elastic/auditbeat/auditbeat/module/file_integrity/metricset_test.go:155
  | Error:      	unexpected path
  | Test:       	TestActions
  | Messages:   	path /private/tmp/TestActions4021485366/001/updated.txt
  | metricset_test.go:155:
  | Error Trace:	/Users/admin/builds/bk-agent-prod-orka-1722331758191225599/elastic/auditbeat/auditbeat/module/file_integrity/metricset_test.go:155
  | Error:      	unexpected path
  | Test:       	TestActions
  | Messages:   	path /private/tmp/TestActions4021485366/002
  |  
  | === FAIL: auditbeat/module/file_integrity TestExcludedFiles (0.17s)
  | metricset_test.go:208:
  | Error Trace:	/Users/admin/builds/bk-agent-prod-orka-1722331758191225599/elastic/auditbeat/auditbeat/module/file_integrity/metricset_test.go:208
  | Error:      	Should be true
  | Test:       	TestExcludedFiles
  | metricset_test.go:208:
  | Error Trace:	/Users/admin/builds/bk-agent-prod-orka-1722331758191225599/elastic/auditbeat/auditbeat/module/file_integrity/metricset_test.go:208
  | Error:      	Should be true
  | Test:       	TestExcludedFiles
  | metricset_test.go:208:
  | Error Trace:	/Users/admin/builds/bk-agent-prod-orka-1722331758191225599/elastic/auditbeat/auditbeat/module/file_integrity/metricset_test.go:208
  | Error:      	Should be true
  | Test:       	TestExcludedFiles
  |  
  | === FAIL: auditbeat/module/file_integrity TestIncludedExcludedFiles (0.16s)
  | metricset_test.go:270:
  | Error Trace:	/Users/admin/builds/bk-agent-prod-orka-1722331758191225599/elastic/auditbeat/auditbeat/module/file_integrity/metricset_test.go:270
  | Error:      	Not equal:
  | expected: map[string]bool{"/tmp/TestIncludedExcludedFiles2240735922/001":true, "/tmp/TestIncludedExcludedFiles2240735922/001/.ssh":true, "/tmp/TestIncludedExcludedFiles2240735922/001/.ssh/known_hosts":true}
  | actual  : map[string]bool{"/private/tmp/TestIncludedExcludedFiles2240735922/001":true, "/private/tmp/TestIncludedExcludedFiles2240735922/001/.ssh":true, "/private/tmp/TestIncludedExcludedFiles2240735922/001/.ssh/known_hosts":true}
  |  
  | Diff:
  | --- Expected
  | +++ Actual
  | @@ -1,5 +1,5 @@
  | (map[string]bool) (len=3) {
  | - (string) (len=44) "/tmp/TestIncludedExcludedFiles2240735922/001": (bool) true,
  | - (string) (len=49) "/tmp/TestIncludedExcludedFiles2240735922/001/.ssh": (bool) true,
  | - (string) (len=61) "/tmp/TestIncludedExcludedFiles2240735922/001/.ssh/known_hosts": (bool) true
  | + (string) (len=52) "/private/tmp/TestIncludedExcludedFiles2240735922/001": (bool) true,
  | + (string) (len=57) "/private/tmp/TestIncludedExcludedFiles2240735922/001/.ssh": (bool) true,
  | + (string) (len=69) "/private/tmp/TestIncludedExcludedFiles2240735922/001/.ssh/known_hosts": (bool) true
  | }
  | Test:       	TestIncludedExcludedFiles

[elasticmachine]

Pinging @elastic/sec-linux-platform (Team:Security-Linux Platform)

[rowlandgeoff]

During the migration of beats-ci from Jenkins to Buildkite, a number of tests were failing consistently due to issues unrelated to the migration. Those tests were disabled to stabilize the CI, with the intent to revisit them post-migration. @oakrizan has reviewed them all in her draft PRs linked above in the description, and has opened tickets such as this one to highlight to the product teams the tests that are currently still disabled and could use some attention.