From 06509704a580ef45ebfc8ff381838f09e78a19dc Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Mon, 11 Feb 2019 23:42:57 -0800 Subject: [PATCH 1/4] Add support for CRI-O based logs autodiscover --- CHANGELOG.asciidoc | 1 + filebeat/autodiscover/builder/hints/config.go | 4 +- .../autodiscover/builder/hints/logs_test.go | 56 +++++++++++++++---- filebeat/input/docker/config.go | 4 ++ filebeat/input/docker/input.go | 31 ++++++++-- 5 files changed, 76 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 005c8261dc5e..e7de86b334c0 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -273,6 +273,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...v7.0.0-beta1[Check the - Added support for ingesting structured Elasticsearch server logs {pull}10428[10428] - Populate more ECS fields in the Suricata module. {pull}10006[10006] - Add module zeek. {issue}9931[9931] {pull}10034[10034] +- Add support for CRI-O based logs autodiscover {pull}10687[10687] *Heartbeat* diff --git a/filebeat/autodiscover/builder/hints/config.go b/filebeat/autodiscover/builder/hints/config.go index 2269daa88dec..2f842eb06abd 100644 --- a/filebeat/autodiscover/builder/hints/config.go +++ b/filebeat/autodiscover/builder/hints/config.go @@ -28,8 +28,8 @@ func defaultConfig() config { rawCfg := map[string]interface{}{ "type": "docker", "containers": map[string]interface{}{ - "ids": []string{ - "${data.container.id}", + "paths": []string{ + "/var/log/containers/${data.kubernetes.pod.name}_${data.kubernetes.namespace}_${data.container.name}-${data.container.id}.log", }, }, } diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index da687231a8a1..e14e03faa697 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -56,6 +56,10 @@ func TestGenerateHints(t *testing.T) { "name": "foobar", "id": "abc", }, + "pod": common.MapStr{ + "name": "pod", + }, + "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -66,7 +70,7 @@ func TestGenerateHints(t *testing.T) { result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "ids": []interface{}{"abc"}, + "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, }, "close_timeout": "true", }, @@ -80,6 +84,10 @@ func TestGenerateHints(t *testing.T) { "name": "foobar", "id": "abc", }, + "pod": common.MapStr{ + "name": "pod", + }, + "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -96,7 +104,7 @@ func TestGenerateHints(t *testing.T) { result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "ids": []interface{}{"abc"}, + "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, }, "include_lines": []interface{}{"^test", "^test1"}, "exclude_lines": []interface{}{"^test2", "^test3"}, @@ -112,6 +120,10 @@ func TestGenerateHints(t *testing.T) { "name": "foobar", "id": "abc", }, + "pod": common.MapStr{ + "name": "pod", + }, + "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -130,7 +142,7 @@ func TestGenerateHints(t *testing.T) { result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "ids": []interface{}{"abc"}, + "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, }, "multiline": map[string]interface{}{ "pattern": "^test", @@ -148,6 +160,10 @@ func TestGenerateHints(t *testing.T) { "name": "foobar", "id": "abc", }, + "pod": common.MapStr{ + "name": "pod", + }, + "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -180,6 +196,10 @@ func TestGenerateHints(t *testing.T) { "name": "foobar", "id": "abc", }, + "pod": common.MapStr{ + "name": "pod", + }, + "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -202,7 +222,7 @@ func TestGenerateHints(t *testing.T) { result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "ids": []interface{}{"abc"}, + "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, }, "close_timeout": "true", "processors": []interface{}{ @@ -226,6 +246,10 @@ func TestGenerateHints(t *testing.T) { "name": "foobar", "id": "abc", }, + "pod": common.MapStr{ + "name": "pod", + }, + "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -246,7 +270,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "all", - "ids": []interface{}{"abc"}, + "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, }, "close_timeout": "true", }, @@ -257,7 +281,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "all", - "ids": []interface{}{"abc"}, + "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, }, "close_timeout": "true", }, @@ -273,6 +297,10 @@ func TestGenerateHints(t *testing.T) { "name": "foobar", "id": "abc", }, + "pod": common.MapStr{ + "name": "pod", + }, + "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -294,7 +322,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "all", - "ids": []interface{}{"abc"}, + "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, }, "close_timeout": "true", }, @@ -305,7 +333,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "all", - "ids": []interface{}{"abc"}, + "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, }, "close_timeout": "true", }, @@ -321,6 +349,10 @@ func TestGenerateHints(t *testing.T) { "name": "foobar", "id": "abc", }, + "pod": common.MapStr{ + "name": "pod", + }, + "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -343,7 +375,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "stdout", - "ids": []interface{}{"abc"}, + "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, }, "close_timeout": "true", }, @@ -354,7 +386,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "stderr", - "ids": []interface{}{"abc"}, + "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, }, "close_timeout": "true", }, @@ -368,8 +400,8 @@ func TestGenerateHints(t *testing.T) { "config": map[string]interface{}{ "type": "docker", "containers": map[string]interface{}{ - "ids": []string{ - "${data.container.id}", + "paths": []string{ + "/var/log/containers/${data.kubernetes.pod.name}_${data.kubernetes.namespace}_${data.container.name}-${data.container.id}.log", }, }, "close_timeout": "true", diff --git a/filebeat/input/docker/config.go b/filebeat/input/docker/config.go index ddd5c1ac6290..9a9377861503 100644 --- a/filebeat/input/docker/config.go +++ b/filebeat/input/docker/config.go @@ -27,6 +27,7 @@ var defaultConfig = config{ } type config struct { + // List of containers' log files to tail Containers containers `config:"containers"` // Partial configures the input to join partial lines @@ -40,9 +41,12 @@ type config struct { } type containers struct { + // IDs + Path to be removed in 8.0 in favor of Paths. IDs []string `config:"ids"` Path string `config:"path"` + Paths []string `config:"paths"` + // Stream can be all, stdout or stderr Stream string `config:"stream"` } diff --git a/filebeat/input/docker/input.go b/filebeat/input/docker/input.go index e762f8f91cca..6a5769701c7c 100644 --- a/filebeat/input/docker/input.go +++ b/filebeat/input/docker/input.go @@ -51,10 +51,19 @@ func NewInput( return nil, errors.Wrap(err, "reading docker input config") } - // Docker input should make sure that no callers should ever pass empty strings as container IDs + // Docker input should make sure that no callers should ever pass empty strings as container IDs or paths // Hence we explicitly make sure that we catch such things and print stack traces in the event of // an invocation so that it can be fixed. - var ids []string + var ids, paths []string + for _, p := range config.Containers.Paths { + if p != "" { + paths = append(paths, p) + } else { + logger.Error("Docker paths can't be empty for Docker input config") + logger.Debugw("Empty path for Docker logfile was received", logp.Stack("stacktrace")) + } + } + for _, containerID := range config.Containers.IDs { if containerID != "" { ids = append(ids, containerID) @@ -64,12 +73,22 @@ func NewInput( } } - if len(ids) == 0 { - return nil, errors.New("Docker input requires at least one entry under 'containers.ids'") + if len(ids) == 0 && len(paths) == 0 { + return nil, errors.New("Docker input requires at least one entry under 'containers.ids' or 'containers.paths'") } - for idx, containerID := range ids { - cfg.SetString("paths", idx, path.Join(config.Containers.Path, containerID, "*.log")) + if len(ids) != 0 && len(paths) != 0 { + return nil, errors.New("can not provide both 'containers.ids' and 'containers.paths' in the same input config") + } + + if len(ids) != 0 { + for idx, containerID := range ids { + cfg.SetString("paths", idx, path.Join(config.Containers.Path, containerID, "*.log")) + } + } else { + for idx, p := range paths { + cfg.SetString("paths", idx, p) + } } if err := checkStream(config.Containers.Stream); err != nil { From 2ec23ca10b313bab2e2e1d43dc2a9adaa4360d05 Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Tue, 12 Feb 2019 11:39:14 -0800 Subject: [PATCH 2/4] Add symlink support and ensure that we use /var/log/pods by default --- filebeat/autodiscover/builder/hints/config.go | 2 +- .../autodiscover/builder/hints/logs_test.go | 40 +++++++++---------- filebeat/input/docker/input.go | 5 +++ 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/filebeat/autodiscover/builder/hints/config.go b/filebeat/autodiscover/builder/hints/config.go index 2f842eb06abd..9da76a95a7fa 100644 --- a/filebeat/autodiscover/builder/hints/config.go +++ b/filebeat/autodiscover/builder/hints/config.go @@ -29,7 +29,7 @@ func defaultConfig() config { "type": "docker", "containers": map[string]interface{}{ "paths": []string{ - "/var/log/containers/${data.kubernetes.pod.name}_${data.kubernetes.namespace}_${data.container.name}-${data.container.id}.log", + "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log", }, }, } diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index e14e03faa697..7a536b68f25a 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -58,8 +58,8 @@ func TestGenerateHints(t *testing.T) { }, "pod": common.MapStr{ "name": "pod", + "uid": "12345", }, - "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -70,7 +70,7 @@ func TestGenerateHints(t *testing.T) { result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, + "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, }, "close_timeout": "true", }, @@ -86,8 +86,8 @@ func TestGenerateHints(t *testing.T) { }, "pod": common.MapStr{ "name": "pod", + "uid": "12345", }, - "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -104,7 +104,7 @@ func TestGenerateHints(t *testing.T) { result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, + "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, }, "include_lines": []interface{}{"^test", "^test1"}, "exclude_lines": []interface{}{"^test2", "^test3"}, @@ -118,12 +118,12 @@ func TestGenerateHints(t *testing.T) { "kubernetes": common.MapStr{ "container": common.MapStr{ "name": "foobar", - "id": "abc", + "uid": "12345", }, "pod": common.MapStr{ "name": "pod", + "uid": "12345", }, - "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -142,7 +142,7 @@ func TestGenerateHints(t *testing.T) { result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, + "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, }, "multiline": map[string]interface{}{ "pattern": "^test", @@ -158,12 +158,10 @@ func TestGenerateHints(t *testing.T) { "kubernetes": common.MapStr{ "container": common.MapStr{ "name": "foobar", - "id": "abc", }, "pod": common.MapStr{ "name": "pod", }, - "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -198,8 +196,8 @@ func TestGenerateHints(t *testing.T) { }, "pod": common.MapStr{ "name": "pod", + "uid": "12345", }, - "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -222,7 +220,7 @@ func TestGenerateHints(t *testing.T) { result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, + "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, }, "close_timeout": "true", "processors": []interface{}{ @@ -248,8 +246,8 @@ func TestGenerateHints(t *testing.T) { }, "pod": common.MapStr{ "name": "pod", + "uid": "12345", }, - "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -270,7 +268,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "all", - "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, + "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, }, "close_timeout": "true", }, @@ -281,7 +279,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "all", - "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, + "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, }, "close_timeout": "true", }, @@ -299,8 +297,8 @@ func TestGenerateHints(t *testing.T) { }, "pod": common.MapStr{ "name": "pod", + "uid": "12345", }, - "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -322,7 +320,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "all", - "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, + "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, }, "close_timeout": "true", }, @@ -333,7 +331,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "all", - "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, + "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, }, "close_timeout": "true", }, @@ -351,8 +349,8 @@ func TestGenerateHints(t *testing.T) { }, "pod": common.MapStr{ "name": "pod", + "uid": "12345", }, - "namespace": "ns", }, "container": common.MapStr{ "name": "foobar", @@ -375,7 +373,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "stdout", - "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, + "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, }, "close_timeout": "true", }, @@ -386,7 +384,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "stderr", - "paths": []interface{}{"/var/log/containers/pod_ns_foobar-abc.log"}, + "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, }, "close_timeout": "true", }, @@ -401,7 +399,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "paths": []string{ - "/var/log/containers/${data.kubernetes.pod.name}_${data.kubernetes.namespace}_${data.container.name}-${data.container.id}.log", + "/var/log/pods/${data.kubernetes.pod.uid}/${data.container.name}/*.log", }, }, "close_timeout": "true", diff --git a/filebeat/input/docker/input.go b/filebeat/input/docker/input.go index 6a5769701c7c..17e763f0b1b1 100644 --- a/filebeat/input/docker/input.go +++ b/filebeat/input/docker/input.go @@ -111,6 +111,11 @@ func NewInput( return nil, errors.Wrap(err, "update input config") } + // Set symlinks to true as CRI-O paths could point to symlinks instead of the actual path. + if err := cfg.SetBool("symlinks", -1, true); err != nil { + return nil, errors.Wrap(err, "update input config") + } + // Add stream to meta to ensure different state per stream if config.Containers.Stream != "all" { if context.Meta == nil { From efd113f8cc59f95bb83dd494d155bd61d55992c2 Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Tue, 12 Feb 2019 22:06:44 -0800 Subject: [PATCH 3/4] Ensuring backward compatibility --- filebeat/autodiscover/builder/hints/config.go | 4 ++- .../autodiscover/builder/hints/logs_test.go | 32 ++++++++++++------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/filebeat/autodiscover/builder/hints/config.go b/filebeat/autodiscover/builder/hints/config.go index 9da76a95a7fa..c52357dcabe9 100644 --- a/filebeat/autodiscover/builder/hints/config.go +++ b/filebeat/autodiscover/builder/hints/config.go @@ -29,7 +29,9 @@ func defaultConfig() config { "type": "docker", "containers": map[string]interface{}{ "paths": []string{ - "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log", + // To be able to use this builder with CRI-O replace paths with: + // /var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log + "/var/lib/docker/containers/${data.kubernetes.container.id}/*-json.log", }, }, } diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index 7a536b68f25a..3caad5d2935a 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -32,6 +32,7 @@ func TestGenerateHints(t *testing.T) { tests := []struct { msg string event bus.Event + path string len int result common.MapStr }{ @@ -44,6 +45,7 @@ func TestGenerateHints(t *testing.T) { }, }, }, + path: "/var/lib/docker/containers/${data.kubernetes.container.id}/*-json.log", len: 0, result: common.MapStr{}, }, @@ -66,11 +68,12 @@ func TestGenerateHints(t *testing.T) { "id": "abc", }, }, - len: 1, + path: "/var/lib/docker/containers/${data.kubernetes.container.id}/*-json.log", + len: 1, result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, + "paths": []interface{}{"/var/lib/docker/containers/abc/*-json.log"}, }, "close_timeout": "true", }, @@ -100,11 +103,12 @@ func TestGenerateHints(t *testing.T) { }, }, }, - len: 1, + len: 1, + path: "/var/lib/docker/containers/${data.kubernetes.container.id}/*-json.log", result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, + "paths": []interface{}{"/var/lib/docker/containers/abc/*-json.log"}, }, "include_lines": []interface{}{"^test", "^test1"}, "exclude_lines": []interface{}{"^test2", "^test3"}, @@ -138,7 +142,8 @@ func TestGenerateHints(t *testing.T) { }, }, }, - len: 1, + len: 1, + path: "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log", result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ @@ -173,7 +178,8 @@ func TestGenerateHints(t *testing.T) { }, }, }, - len: 1, + len: 1, + path: "", result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ @@ -216,7 +222,8 @@ func TestGenerateHints(t *testing.T) { }, }, }, - len: 1, + len: 1, + path: "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log", result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ @@ -259,7 +266,8 @@ func TestGenerateHints(t *testing.T) { }, }, }, - len: 1, + len: 1, + path: "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log", result: common.MapStr{ "module": "apache2", "error": map[string]interface{}{ @@ -311,7 +319,8 @@ func TestGenerateHints(t *testing.T) { }, }, }, - len: 1, + len: 1, + path: "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log", result: common.MapStr{ "module": "apache2", "access": map[string]interface{}{ @@ -364,7 +373,8 @@ func TestGenerateHints(t *testing.T) { }, }, }, - len: 1, + len: 1, + path: "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log", result: common.MapStr{ "module": "apache2", "access": map[string]interface{}{ @@ -399,7 +409,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "paths": []string{ - "/var/log/pods/${data.kubernetes.pod.uid}/${data.container.name}/*.log", + test.path, }, }, "close_timeout": "true", From 2f9814a698a2801a72fe08a6988415977ea018f5 Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Thu, 14 Feb 2019 08:25:43 -0800 Subject: [PATCH 4/4] Incorporate review comments --- filebeat/autodiscover/builder/hints/config.go | 2 +- .../autodiscover/builder/hints/logs_test.go | 304 ++++++++++++++---- filebeat/input/docker/config.go | 1 - filebeat/input/docker/input.go | 9 +- 4 files changed, 255 insertions(+), 61 deletions(-) diff --git a/filebeat/autodiscover/builder/hints/config.go b/filebeat/autodiscover/builder/hints/config.go index c52357dcabe9..cae7aa2192cd 100644 --- a/filebeat/autodiscover/builder/hints/config.go +++ b/filebeat/autodiscover/builder/hints/config.go @@ -31,7 +31,7 @@ func defaultConfig() config { "paths": []string{ // To be able to use this builder with CRI-O replace paths with: // /var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log - "/var/lib/docker/containers/${data.kubernetes.container.id}/*-json.log", + "/var/lib/docker/containers/${data.container.id}/*-json.log", }, }, } diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index 3caad5d2935a..de56e777765a 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -32,7 +32,6 @@ func TestGenerateHints(t *testing.T) { tests := []struct { msg string event bus.Event - path string len int result common.MapStr }{ @@ -45,7 +44,6 @@ func TestGenerateHints(t *testing.T) { }, }, }, - path: "/var/lib/docker/containers/${data.kubernetes.container.id}/*-json.log", len: 0, result: common.MapStr{}, }, @@ -58,22 +56,17 @@ func TestGenerateHints(t *testing.T) { "name": "foobar", "id": "abc", }, - "pod": common.MapStr{ - "name": "pod", - "uid": "12345", - }, }, "container": common.MapStr{ "name": "foobar", "id": "abc", }, }, - path: "/var/lib/docker/containers/${data.kubernetes.container.id}/*-json.log", - len: 1, + len: 1, result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "paths": []interface{}{"/var/lib/docker/containers/abc/*-json.log"}, + "ids": []interface{}{"abc"}, }, "close_timeout": "true", }, @@ -87,10 +80,6 @@ func TestGenerateHints(t *testing.T) { "name": "foobar", "id": "abc", }, - "pod": common.MapStr{ - "name": "pod", - "uid": "12345", - }, }, "container": common.MapStr{ "name": "foobar", @@ -103,12 +92,11 @@ func TestGenerateHints(t *testing.T) { }, }, }, - len: 1, - path: "/var/lib/docker/containers/${data.kubernetes.container.id}/*-json.log", + len: 1, result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "paths": []interface{}{"/var/lib/docker/containers/abc/*-json.log"}, + "ids": []interface{}{"abc"}, }, "include_lines": []interface{}{"^test", "^test1"}, "exclude_lines": []interface{}{"^test2", "^test3"}, @@ -122,11 +110,7 @@ func TestGenerateHints(t *testing.T) { "kubernetes": common.MapStr{ "container": common.MapStr{ "name": "foobar", - "uid": "12345", - }, - "pod": common.MapStr{ - "name": "pod", - "uid": "12345", + "id": "abc", }, }, "container": common.MapStr{ @@ -142,12 +126,11 @@ func TestGenerateHints(t *testing.T) { }, }, }, - len: 1, - path: "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log", + len: 1, result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, + "ids": []interface{}{"abc"}, }, "multiline": map[string]interface{}{ "pattern": "^test", @@ -163,9 +146,7 @@ func TestGenerateHints(t *testing.T) { "kubernetes": common.MapStr{ "container": common.MapStr{ "name": "foobar", - }, - "pod": common.MapStr{ - "name": "pod", + "id": "abc", }, }, "container": common.MapStr{ @@ -178,8 +159,7 @@ func TestGenerateHints(t *testing.T) { }, }, }, - len: 1, - path: "", + len: 1, result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ @@ -200,10 +180,6 @@ func TestGenerateHints(t *testing.T) { "name": "foobar", "id": "abc", }, - "pod": common.MapStr{ - "name": "pod", - "uid": "12345", - }, }, "container": common.MapStr{ "name": "foobar", @@ -222,12 +198,11 @@ func TestGenerateHints(t *testing.T) { }, }, }, - len: 1, - path: "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log", + len: 1, result: common.MapStr{ "type": "docker", "containers": map[string]interface{}{ - "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, + "ids": []interface{}{"abc"}, }, "close_timeout": "true", "processors": []interface{}{ @@ -251,10 +226,6 @@ func TestGenerateHints(t *testing.T) { "name": "foobar", "id": "abc", }, - "pod": common.MapStr{ - "name": "pod", - "uid": "12345", - }, }, "container": common.MapStr{ "name": "foobar", @@ -266,8 +237,7 @@ func TestGenerateHints(t *testing.T) { }, }, }, - len: 1, - path: "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log", + len: 1, result: common.MapStr{ "module": "apache2", "error": map[string]interface{}{ @@ -276,7 +246,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "all", - "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, + "ids": []interface{}{"abc"}, }, "close_timeout": "true", }, @@ -287,7 +257,7 @@ func TestGenerateHints(t *testing.T) { "type": "docker", "containers": map[string]interface{}{ "stream": "all", - "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, + "ids": []interface{}{"abc"}, }, "close_timeout": "true", }, @@ -296,6 +266,179 @@ func TestGenerateHints(t *testing.T) { }, { msg: "Hint with module should honor defined filesets", + event: bus.Event{ + "host": "1.2.3.4", + "kubernetes": common.MapStr{ + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + "hints": common.MapStr{ + "logs": common.MapStr{ + "module": "apache2", + "fileset": "access", + }, + }, + }, + len: 1, + result: common.MapStr{ + "module": "apache2", + "access": map[string]interface{}{ + "enabled": true, + "input": map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "stream": "all", + "ids": []interface{}{"abc"}, + }, + "close_timeout": "true", + }, + }, + "error": map[string]interface{}{ + "enabled": false, + "input": map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "stream": "all", + "ids": []interface{}{"abc"}, + }, + "close_timeout": "true", + }, + }, + }, + }, + { + msg: "Hint with module should honor defined filesets with streams", + event: bus.Event{ + "host": "1.2.3.4", + "kubernetes": common.MapStr{ + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + "hints": common.MapStr{ + "logs": common.MapStr{ + "module": "apache2", + "fileset.stdout": "access", + "fileset.stderr": "error", + }, + }, + }, + len: 1, + result: common.MapStr{ + "module": "apache2", + "access": map[string]interface{}{ + "enabled": true, + "input": map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "stream": "stdout", + "ids": []interface{}{"abc"}, + }, + "close_timeout": "true", + }, + }, + "error": map[string]interface{}{ + "enabled": true, + "input": map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "stream": "stderr", + "ids": []interface{}{"abc"}, + }, + "close_timeout": "true", + }, + }, + }, + }, + } + + for _, test := range tests { + cfg, _ := common.NewConfigFrom(map[string]interface{}{ + "config": map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "ids": []string{ + "${data.container.id}", + }, + }, + "close_timeout": "true", + }, + }) + + // Configure path for modules access + abs, _ := filepath.Abs("../../..") + err := paths.InitPaths(&paths.Path{ + Home: abs, + }) + + l, err := NewLogHints(cfg) + if err != nil { + t.Fatal(err) + } + + cfgs := l.CreateConfig(test.event) + assert.Equal(t, len(cfgs), test.len, test.msg) + if test.len != 0 { + config := common.MapStr{} + err := cfgs[0].Unpack(&config) + assert.Nil(t, err, test.msg) + + assert.Equal(t, test.result, config, test.msg) + } + + } +} + +func TestGenerateHintsWithPaths(t *testing.T) { + tests := []struct { + msg string + event bus.Event + path string + len int + result common.MapStr + }{ + { + msg: "Empty event hints should return default config", + event: bus.Event{ + "host": "1.2.3.4", + "kubernetes": common.MapStr{ + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + "pod": common.MapStr{ + "name": "pod", + "uid": "12345", + }, + }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + }, + path: "/var/lib/docker/containers/${data.kubernetes.container.id}/*-json.log", + len: 1, + result: common.MapStr{ + "type": "docker", + "containers": map[string]interface{}{ + "paths": []interface{}{"/var/lib/docker/containers/abc/*-json.log"}, + }, + "close_timeout": "true", + }, + }, + { + msg: "Hint with processors config must have a processors in the input config", event: bus.Event{ "host": "1.2.3.4", "kubernetes": common.MapStr{ @@ -314,8 +457,58 @@ func TestGenerateHints(t *testing.T) { }, "hints": common.MapStr{ "logs": common.MapStr{ - "module": "apache2", - "fileset": "access", + "processors": common.MapStr{ + "1": common.MapStr{ + "dissect": common.MapStr{ + "tokenizer": "%{key1} %{key2}", + }, + }, + "drop_event": common.MapStr{}, + }, + }, + }, + }, + len: 1, + path: "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log", + result: common.MapStr{ + "type": "docker", + "containers": map[string]interface{}{ + "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, + }, + "close_timeout": "true", + "processors": []interface{}{ + map[string]interface{}{ + "dissect": map[string]interface{}{ + "tokenizer": "%{key1} %{key2}", + }, + }, + map[string]interface{}{ + "drop_event": nil, + }, + }, + }, + }, + { + msg: "Hint with module should attach input to its filesets", + event: bus.Event{ + "host": "1.2.3.4", + "kubernetes": common.MapStr{ + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + "pod": common.MapStr{ + "name": "pod", + "uid": "12345", + }, + }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + "hints": common.MapStr{ + "logs": common.MapStr{ + "module": "apache2", }, }, }, @@ -323,7 +516,7 @@ func TestGenerateHints(t *testing.T) { path: "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log", result: common.MapStr{ "module": "apache2", - "access": map[string]interface{}{ + "error": map[string]interface{}{ "enabled": true, "input": map[string]interface{}{ "type": "docker", @@ -334,8 +527,8 @@ func TestGenerateHints(t *testing.T) { "close_timeout": "true", }, }, - "error": map[string]interface{}{ - "enabled": false, + "access": map[string]interface{}{ + "enabled": true, "input": map[string]interface{}{ "type": "docker", "containers": map[string]interface{}{ @@ -348,7 +541,7 @@ func TestGenerateHints(t *testing.T) { }, }, { - msg: "Hint with module should honor defined filesets with streams", + msg: "Hint with module should honor defined filesets", event: bus.Event{ "host": "1.2.3.4", "kubernetes": common.MapStr{ @@ -367,9 +560,8 @@ func TestGenerateHints(t *testing.T) { }, "hints": common.MapStr{ "logs": common.MapStr{ - "module": "apache2", - "fileset.stdout": "access", - "fileset.stderr": "error", + "module": "apache2", + "fileset": "access", }, }, }, @@ -382,18 +574,18 @@ func TestGenerateHints(t *testing.T) { "input": map[string]interface{}{ "type": "docker", "containers": map[string]interface{}{ - "stream": "stdout", + "stream": "all", "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, }, "close_timeout": "true", }, }, "error": map[string]interface{}{ - "enabled": true, + "enabled": false, "input": map[string]interface{}{ "type": "docker", "containers": map[string]interface{}{ - "stream": "stderr", + "stream": "all", "paths": []interface{}{"/var/log/pods/12345/foobar/*.log"}, }, "close_timeout": "true", diff --git a/filebeat/input/docker/config.go b/filebeat/input/docker/config.go index 9a9377861503..2f0b1323c7eb 100644 --- a/filebeat/input/docker/config.go +++ b/filebeat/input/docker/config.go @@ -41,7 +41,6 @@ type config struct { } type containers struct { - // IDs + Path to be removed in 8.0 in favor of Paths. IDs []string `config:"ids"` Path string `config:"path"` diff --git a/filebeat/input/docker/input.go b/filebeat/input/docker/input.go index 17e763f0b1b1..458745e66dd9 100644 --- a/filebeat/input/docker/input.go +++ b/filebeat/input/docker/input.go @@ -77,6 +77,7 @@ func NewInput( return nil, errors.New("Docker input requires at least one entry under 'containers.ids' or 'containers.paths'") } + // IDs + Path and Paths are mutually exclusive. Ensure that only one of them are set in a given configuration if len(ids) != 0 && len(paths) != 0 { return nil, errors.New("can not provide both 'containers.ids' and 'containers.paths' in the same input config") } @@ -111,9 +112,11 @@ func NewInput( return nil, errors.Wrap(err, "update input config") } - // Set symlinks to true as CRI-O paths could point to symlinks instead of the actual path. - if err := cfg.SetBool("symlinks", -1, true); err != nil { - return nil, errors.Wrap(err, "update input config") + if len(paths) != 0 { + // Set symlinks to true as CRI-O paths could point to symlinks instead of the actual path. + if err := cfg.SetBool("symlinks", -1, true); err != nil { + return nil, errors.Wrap(err, "update input config") + } } // Add stream to meta to ensure different state per stream