From 80b1927386cf87ba297133f1547d5c0d4d483cd2 Mon Sep 17 00:00:00 2001 From: Jako Tinkus Date: Tue, 30 Jun 2020 16:51:07 +0300 Subject: [PATCH 1/4] Initial container id fix --- .../add_process_metadata.go | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/libbeat/processors/add_process_metadata/add_process_metadata.go b/libbeat/processors/add_process_metadata/add_process_metadata.go index 4aff1df85768..dc04baf9c777 100644 --- a/libbeat/processors/add_process_metadata/add_process_metadata.go +++ b/libbeat/processors/add_process_metadata/add_process_metadata.go @@ -20,6 +20,7 @@ package add_process_metadata import ( "fmt" "strconv" + "strings" "time" "github.com/pkg/errors" @@ -190,19 +191,43 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul return nil, errors.Errorf("cannot parse field '%s' (not an integer or string)", pidField) } + var meta common.MapStr + metaPtr, err := p.provider.GetProcessMetadata(pid) if err != nil || metaPtr == nil { + // no process metadata, lets still try for contianer id p.log.Debugf("failed to get process metadata for PID=%d: %v", pid, err) + meta = common.MapStr{} + } else { + meta = metaPtr.fields + } + + cid, err := p.getContainerID(pid) + if err != nil { + p.log.Debugf("failed to get container id for PID=%d: %v", pid, err) + } else { + if _, err = meta.Put("container", common.MapStr{"id": cid}); err != nil { + return nil, err + } + } + + // no metadata nor container id + if len(meta) == 0 { return nil, ErrNoProcess } - meta := metaPtr.fields - if err = p.enrichContainerID(pid, meta); err != nil { - return nil, err + mappings := p.mappings.Clone() + + if b, _ := meta.HasKey("container"); b && len(meta) == 1 { + for key := range mappings { + if !strings.Contains(key, "container.id") { + mappings.Delete(key) + } + } } result = event.Clone() - for dest, sourceIf := range p.mappings { + for dest, sourceIf := range mappings { source, castOk := sourceIf.(string) if !castOk { // Should never happen, as source is generated by Config.prepareMappings() @@ -228,19 +253,16 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul return result, nil } -// enrichContainerID adds container.id into meta for mapping to pickup -func (p *addProcessMetadata) enrichContainerID(pid int, meta common.MapStr) error { +func (p *addProcessMetadata) getContainerID(pid int) (string, error) { if p.cidProvider == nil { - return nil + return "", nil } cid, err := p.cidProvider.GetCid(pid) if err != nil { - return err - } - if _, err = meta.Put("container", common.MapStr{"id": cid}); err != nil { - return err + return "", err + } else { + return cid, nil } - return nil } // String returns the processor representation formatted as a string From b1ba706deff35a1cf7a2e097e00a74882cff0ee0 Mon Sep 17 00:00:00 2001 From: Jako Tinkus Date: Thu, 9 Jul 2020 10:28:16 +0300 Subject: [PATCH 2/4] Polished code and added tests --- .../add_process_metadata.go | 26 ++---- .../add_process_metadata_test.go | 86 ++++++++++++++++++- 2 files changed, 92 insertions(+), 20 deletions(-) diff --git a/libbeat/processors/add_process_metadata/add_process_metadata.go b/libbeat/processors/add_process_metadata/add_process_metadata.go index dc04baf9c777..847ed6fe887d 100644 --- a/libbeat/processors/add_process_metadata/add_process_metadata.go +++ b/libbeat/processors/add_process_metadata/add_process_metadata.go @@ -20,7 +20,6 @@ package add_process_metadata import ( "fmt" "strconv" - "strings" "time" "github.com/pkg/errors" @@ -195,7 +194,7 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul metaPtr, err := p.provider.GetProcessMetadata(pid) if err != nil || metaPtr == nil { - // no process metadata, lets still try for contianer id + // no process metadata, lets still try to get contianer id p.log.Debugf("failed to get process metadata for PID=%d: %v", pid, err) meta = common.MapStr{} } else { @@ -203,7 +202,7 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul } cid, err := p.getContainerID(pid) - if err != nil { + if cid == "" || err != nil { p.log.Debugf("failed to get container id for PID=%d: %v", pid, err) } else { if _, err = meta.Put("container", common.MapStr{"id": cid}); err != nil { @@ -211,23 +210,13 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul } } - // no metadata nor container id if len(meta) == 0 { + // no metadata nor container id return nil, ErrNoProcess } - mappings := p.mappings.Clone() - - if b, _ := meta.HasKey("container"); b && len(meta) == 1 { - for key := range mappings { - if !strings.Contains(key, "container.id") { - mappings.Delete(key) - } - } - } - result = event.Clone() - for dest, sourceIf := range mappings { + for dest, sourceIf := range p.mappings { source, castOk := sourceIf.(string) if !castOk { // Should never happen, as source is generated by Config.prepareMappings() @@ -241,8 +230,8 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul value, err := meta.GetValue(source) if err != nil { - // Should never happen - return nil, err + // skip missing values + continue } if _, err = result.Put(dest, value); err != nil { @@ -260,9 +249,8 @@ func (p *addProcessMetadata) getContainerID(pid int) (string, error) { cid, err := p.cidProvider.GetCid(pid) if err != nil { return "", err - } else { - return cid, nil } + return cid, nil } // String returns the processor representation formatted as a string diff --git a/libbeat/processors/add_process_metadata/add_process_metadata_test.go b/libbeat/processors/add_process_metadata/add_process_metadata_test.go index 8bb2cd4b6259..f9b4aaa681c4 100644 --- a/libbeat/processors/add_process_metadata/add_process_metadata_test.go +++ b/libbeat/processors/add_process_metadata/add_process_metadata_test.go @@ -49,12 +49,42 @@ func TestAddProcessMetadata(t *testing.T) { ppid: 0, startTime: startTime, }, + 3: { + name: "systemd", + title: "/usr/lib/systemd/systemd --switched-root --system --deserialize 22", + exe: "/usr/lib/systemd/systemd", + args: []string{"/usr/lib/systemd/systemd", "--switched-root", "--system", "--deserialize", "22"}, + env: map[string]string{ + "HOME": "/", + "TERM": "linux", + "BOOT_IMAGE": "/boot/vmlinuz-4.11.8-300.fc26.x86_64", + "LANG": "en_US.UTF-8", + }, + pid: 1, + ppid: 0, + startTime: startTime, + }, } // mock of the cgroup processCgroupPaths processCgroupPaths = func(_ string, pid int) (map[string]string, error) { testMap := map[int]map[string]string{ - 1: map[string]string{ + 1: { + "cpu": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "net_prio": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "blkio": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "perf_event": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "freezer": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "pids": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "hugetlb": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "cpuacct": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "cpuset": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "net_cls": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "devices": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "memory": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "name=systemd": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + }, + 2: { "cpu": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", "net_prio": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", "blkio": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", @@ -510,6 +540,60 @@ func TestAddProcessMetadata(t *testing.T) { }, }, }, + { + description: "no process metadata available", + config: common.MapStr{ + "match_pids": []string{"system.process.ppid"}, + "cgroup_regex": "\\/.+\\/.+\\/.+\\/([0-9a-f]{64}).*", + }, + event: common.MapStr{ + "system": common.MapStr{ + "process": common.MapStr{ + "ppid": "2", + }, + }, + }, + expected: common.MapStr{ + "system": common.MapStr{ + "process": common.MapStr{ + "ppid": "2", + }, + }, + "container": common.MapStr{ + "id": "b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + }, + }, + }, + { + description: "no container id available", + config: common.MapStr{ + "match_pids": []string{"system.process.ppid"}, + "cgroup_regex": "\\/.+\\/.+\\/.+\\/([0-9a-f]{64}).*", + }, + event: common.MapStr{ + "system": common.MapStr{ + "process": common.MapStr{ + "ppid": "3", + }, + }, + }, + expected: common.MapStr{ + "system": common.MapStr{ + "process": common.MapStr{ + "ppid": "3", + }, + }, + "process": common.MapStr{ + "name": "systemd", + "title": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22", + "executable": "/usr/lib/systemd/systemd", + "args": []string{"/usr/lib/systemd/systemd", "--switched-root", "--system", "--deserialize", "22"}, + "pid": 1, + "ppid": 0, + "start_time": startTime, + }, + }, + }, { description: "without cgroup cache", config: common.MapStr{ From 84d7cad8b8c3b21565276b8965b0251b4f5102a1 Mon Sep 17 00:00:00 2001 From: Jako Tinkus Date: Thu, 9 Jul 2020 10:42:23 +0300 Subject: [PATCH 3/4] Fix typo --- libbeat/processors/add_process_metadata/add_process_metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/processors/add_process_metadata/add_process_metadata.go b/libbeat/processors/add_process_metadata/add_process_metadata.go index 847ed6fe887d..c41ca9a73d61 100644 --- a/libbeat/processors/add_process_metadata/add_process_metadata.go +++ b/libbeat/processors/add_process_metadata/add_process_metadata.go @@ -194,7 +194,7 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul metaPtr, err := p.provider.GetProcessMetadata(pid) if err != nil || metaPtr == nil { - // no process metadata, lets still try to get contianer id + // no process metadata, lets still try to get container id p.log.Debugf("failed to get process metadata for PID=%d: %v", pid, err) meta = common.MapStr{} } else { From fc50aa2f59d9ab43a3233427b2b15f065bcdd4d0 Mon Sep 17 00:00:00 2001 From: Jako Tinkus Date: Mon, 3 Aug 2020 15:16:42 +0300 Subject: [PATCH 4/4] Update changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c12c2a9574f3..111d2d1db3c9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -364,6 +364,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562] - Add support for DNS over TLS for the dns_processor. {pull}19321[19321] - Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215] +- Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767] *Auditbeat*