add_process_metadata processor adds container id even if process meta…

…data not accessible (elastic#19767)
ChrsMark · Aug 4, 2020 · 99191e9 · 99191e9
1 parent aaeead0
commit 99191e9
Show file tree

Hide file tree

Showing 3 changed files with 110 additions and 15 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -364,6 +364,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562]
 - Add support for DNS over TLS for the dns_processor. {pull}19321[19321]
 - Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215]
+- Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767]
 
 *Auditbeat*
 

diff --git a/libbeat/processors/add_process_metadata/add_process_metadata.go b/libbeat/processors/add_process_metadata/add_process_metadata.go
@@ -190,15 +190,29 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul
 		return nil, errors.Errorf("cannot parse field '%s' (not an integer or string)", pidField)
 	}
 
+	var meta common.MapStr
+
 	metaPtr, err := p.provider.GetProcessMetadata(pid)
 	if err != nil || metaPtr == nil {
+		// no process metadata, lets still try to get container id
 		p.log.Debugf("failed to get process metadata for PID=%d: %v", pid, err)
-		return nil, ErrNoProcess
+		meta = common.MapStr{}
+	} else {
+		meta = metaPtr.fields
 	}
-	meta := metaPtr.fields
 
-	if err = p.enrichContainerID(pid, meta); err != nil {
-		return nil, err
+	cid, err := p.getContainerID(pid)
+	if cid == "" || err != nil {
+		p.log.Debugf("failed to get container id for PID=%d: %v", pid, err)
+	} else {
+		if _, err = meta.Put("container", common.MapStr{"id": cid}); err != nil {
+			return nil, err
+		}
+	}
+
+	if len(meta) == 0 {
+		// no metadata nor container id
+		return nil, ErrNoProcess
 	}
 
 	result = event.Clone()
@@ -216,8 +230,8 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul
 
 		value, err := meta.GetValue(source)
 		if err != nil {
-			// Should never happen
-			return nil, err
+			// skip missing values
+			continue
 		}
 
 		if _, err = result.Put(dest, value); err != nil {
@@ -228,19 +242,15 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul
 	return result, nil
 }
 
-// enrichContainerID adds container.id into meta for mapping to pickup
-func (p *addProcessMetadata) enrichContainerID(pid int, meta common.MapStr) error {
+func (p *addProcessMetadata) getContainerID(pid int) (string, error) {
 	if p.cidProvider == nil {
-		return nil
+		return "", nil
 	}
 	cid, err := p.cidProvider.GetCid(pid)
 	if err != nil {
-		return err
-	}
-	if _, err = meta.Put("container", common.MapStr{"id": cid}); err != nil {
-		return err
+		return "", err
 	}
-	return nil
+	return cid, nil
 }
 
 // String returns the processor representation formatted as a string

diff --git a/libbeat/processors/add_process_metadata/add_process_metadata_test.go b/libbeat/processors/add_process_metadata/add_process_metadata_test.go
@@ -49,12 +49,42 @@ func TestAddProcessMetadata(t *testing.T) {
 			ppid:      0,
 			startTime: startTime,
 		},
+		3: {
+			name:  "systemd",
+			title: "/usr/lib/systemd/systemd --switched-root --system --deserialize 22",
+			exe:   "/usr/lib/systemd/systemd",
+			args:  []string{"/usr/lib/systemd/systemd", "--switched-root", "--system", "--deserialize", "22"},
+			env: map[string]string{
+				"HOME":       "/",
+				"TERM":       "linux",
+				"BOOT_IMAGE": "/boot/vmlinuz-4.11.8-300.fc26.x86_64",
+				"LANG":       "en_US.UTF-8",
+			},
+			pid:       1,
+			ppid:      0,
+			startTime: startTime,
+		},
 	}
 
 	// mock of the cgroup processCgroupPaths
 	processCgroupPaths = func(_ string, pid int) (map[string]string, error) {
 		testMap := map[int]map[string]string{
-			1: map[string]string{
+			1: {
+				"cpu":          "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+				"net_prio":     "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+				"blkio":        "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+				"perf_event":   "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+				"freezer":      "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+				"pids":         "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+				"hugetlb":      "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+				"cpuacct":      "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+				"cpuset":       "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+				"net_cls":      "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+				"devices":      "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+				"memory":       "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+				"name=systemd": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+			},
+			2: {
 				"cpu":          "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
 				"net_prio":     "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
 				"blkio":        "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
@@ -510,6 +540,60 @@ func TestAddProcessMetadata(t *testing.T) {
 				},
 			},
 		},
+		{
+			description: "no process metadata available",
+			config: common.MapStr{
+				"match_pids":   []string{"system.process.ppid"},
+				"cgroup_regex": "\\/.+\\/.+\\/.+\\/([0-9a-f]{64}).*",
+			},
+			event: common.MapStr{
+				"system": common.MapStr{
+					"process": common.MapStr{
+						"ppid": "2",
+					},
+				},
+			},
+			expected: common.MapStr{
+				"system": common.MapStr{
+					"process": common.MapStr{
+						"ppid": "2",
+					},
+				},
+				"container": common.MapStr{
+					"id": "b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1",
+				},
+			},
+		},
+		{
+			description: "no container id available",
+			config: common.MapStr{
+				"match_pids":   []string{"system.process.ppid"},
+				"cgroup_regex": "\\/.+\\/.+\\/.+\\/([0-9a-f]{64}).*",
+			},
+			event: common.MapStr{
+				"system": common.MapStr{
+					"process": common.MapStr{
+						"ppid": "3",
+					},
+				},
+			},
+			expected: common.MapStr{
+				"system": common.MapStr{
+					"process": common.MapStr{
+						"ppid": "3",
+					},
+				},
+				"process": common.MapStr{
+					"name":       "systemd",
+					"title":      "/usr/lib/systemd/systemd --switched-root --system --deserialize 22",
+					"executable": "/usr/lib/systemd/systemd",
+					"args":       []string{"/usr/lib/systemd/systemd", "--switched-root", "--system", "--deserialize", "22"},
+					"pid":        1,
+					"ppid":       0,
+					"start_time": startTime,
+				},
+			},
+		},
 		{
 			description: "without cgroup cache",
 			config: common.MapStr{