From ba0f804e986bf900d8ae3a008a00bb3cb232114f Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Thu, 23 Dec 2021 10:59:59 +0100 Subject: [PATCH 1/5] align elastic-agent-standalone manifest with the managed version Signed-off-by: Tetiana Kravchenko --- .../elastic-agent-standalone-kubernetes.yaml | 85 ++++++++++++++++--- ...-agent-standalone-daemonset-configmap.yaml | 79 +++++++++++++++-- 2 files changed, 147 insertions(+), 17 deletions(-) diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml index 6b306051eb2..44c69d6232f 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml @@ -32,7 +32,7 @@ data: meta: package: name: kubernetes - version: 0.2.8 + version: 1.9.0 data_stream: namespace: default streams: @@ -54,6 +54,7 @@ data: - event period: 10s add_metadata: true + skip_older: true - data_stream: dataset: kubernetes.state_container type: metrics @@ -72,6 +73,15 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + - data_stream: + dataset: kubernetes.state_daemonset + type: metrics + metricsets: + - state_daemonset + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s - data_stream: dataset: kubernetes.state_deployment type: metrics @@ -214,20 +224,75 @@ data: fields: ecs.version: 1.12.0 - name: container-log - type: logfile + type: filestream use_output: default meta: package: - name: log - version: 0.4.6 + name: kubernetes + version: 1.9.0 data_stream: namespace: default streams: - data_stream: - dataset: generic - symlinks: true + dataset: kubernetes.container_logs + type: logs + prospector.scanner.symlinks: true + parsers: + - container: + stream: all + format: auto + # - ndjson: + # target: json + # - multiline: + # type: pattern + # pattern: '^\[' + # negate: true + # match: after paths: - /var/log/containers/*${kubernetes.container.id}.log + - name: audit-log + type: filestream + use_output: default + meta: + package: + name: kubernetes + version: 1.9.0 + data_stream: + namespace: default + streams: + - data_stream: + dataset: kubernetes.audit_logs + type: logs + exclude_files: + - .gz$ + parsers: + - ndjson: + add_error_key: true + target: kubernetes_audit + paths: + - /var/log/kubernetes/kube-apiserver-audit.log + processors: + - rename: + fields: + - from: kubernetes_audit + to: kubernetes.audit + - script: + id: dedot_annotations + lang: javascript + source: | + function process(event) { + var audit = event.Get("kubernetes.audit"); + for (var annotation in audit["annotations"]) { + var annotation_dedoted = annotation.replace(/\./g,'_') + event.Rename("kubernetes.audit.annotations."+annotation, "kubernetes.audit.annotations."+annotation_dedoted) + } + return event; + } function test() { + var event = process(new Event({ "kubernetes": { "audit": { "annotations": { "authorization.k8s.io/decision": "allow", "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:kube-scheduler\" of ClusterRole \"system:kube-scheduler\" to User \"system:kube-scheduler\"" } } } })); + if (event.Get("kubernetes.audit.annotations.authorization_k8s_io/decision") !== "allow") { + throw "expected kubernetes.audit.annotations.authorization_k8s_io/decision === allow"; + } + } - name: system-metrics type: system/metrics use_output: default @@ -332,7 +397,7 @@ data: meta: package: name: kubernetes - version: 0.2.8 + version: 1.9.0 data_stream: namespace: default streams: @@ -471,7 +536,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: elastic-agent - image: docker.elastic.co/beats/elastic-agent:8.0.0 + image: docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT args: [ "-c", "/etc/agent.yml", "-e", @@ -480,9 +545,9 @@ spec: - name: ES_USERNAME value: "elastic" - name: ES_PASSWORD - value: "" + value: "changeme" - name: ES_HOST - value: "" + value: "elastic-package-stack_elasticsearch_1" - name: NODE_NAME valueFrom: fieldRef: diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml index d3f290b2aab..3c1306456de 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml @@ -32,7 +32,7 @@ data: meta: package: name: kubernetes - version: 0.2.8 + version: 1.9.0 data_stream: namespace: default streams: @@ -54,6 +54,7 @@ data: - event period: 10s add_metadata: true + skip_older: true - data_stream: dataset: kubernetes.state_container type: metrics @@ -72,6 +73,15 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + - data_stream: + dataset: kubernetes.state_daemonset + type: metrics + metricsets: + - state_daemonset + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s - data_stream: dataset: kubernetes.state_deployment type: metrics @@ -214,20 +224,75 @@ data: fields: ecs.version: 1.12.0 - name: container-log - type: logfile + type: filestream use_output: default meta: package: - name: log - version: 0.4.6 + name: kubernetes + version: 1.9.0 data_stream: namespace: default streams: - data_stream: - dataset: generic - symlinks: true + dataset: kubernetes.container_logs + type: logs + prospector.scanner.symlinks: true + parsers: + - container: + stream: all + format: auto + # - ndjson: + # target: json + # - multiline: + # type: pattern + # pattern: '^\[' + # negate: true + # match: after paths: - /var/log/containers/*${kubernetes.container.id}.log + - name: audit-log + type: filestream + use_output: default + meta: + package: + name: kubernetes + version: 1.9.0 + data_stream: + namespace: default + streams: + - data_stream: + dataset: kubernetes.audit_logs + type: logs + exclude_files: + - .gz$ + parsers: + - ndjson: + add_error_key: true + target: kubernetes_audit + paths: + - /var/log/kubernetes/kube-apiserver-audit.log + processors: + - rename: + fields: + - from: kubernetes_audit + to: kubernetes.audit + - script: + id: dedot_annotations + lang: javascript + source: | + function process(event) { + var audit = event.Get("kubernetes.audit"); + for (var annotation in audit["annotations"]) { + var annotation_dedoted = annotation.replace(/\./g,'_') + event.Rename("kubernetes.audit.annotations."+annotation, "kubernetes.audit.annotations."+annotation_dedoted) + } + return event; + } function test() { + var event = process(new Event({ "kubernetes": { "audit": { "annotations": { "authorization.k8s.io/decision": "allow", "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:kube-scheduler\" of ClusterRole \"system:kube-scheduler\" to User \"system:kube-scheduler\"" } } } })); + if (event.Get("kubernetes.audit.annotations.authorization_k8s_io/decision") !== "allow") { + throw "expected kubernetes.audit.annotations.authorization_k8s_io/decision === allow"; + } + } - name: system-metrics type: system/metrics use_output: default @@ -332,7 +397,7 @@ data: meta: package: name: kubernetes - version: 0.2.8 + version: 1.9.0 data_stream: namespace: default streams: From 83a7432a75a569b27bc5f05692e61a0e8924f02d Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Thu, 23 Dec 2021 11:34:08 +0100 Subject: [PATCH 2/5] revetn docker image version Signed-off-by: Tetiana Kravchenko --- deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml index 44c69d6232f..c7bd222c730 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml @@ -536,7 +536,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: elastic-agent - image: docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT + image: docker.elastic.co/beats/elastic-agent:8.0.0 args: [ "-c", "/etc/agent.yml", "-e", @@ -545,7 +545,7 @@ spec: - name: ES_USERNAME value: "elastic" - name: ES_PASSWORD - value: "changeme" + value: "" - name: ES_HOST value: "elastic-package-stack_elasticsearch_1" - name: NODE_NAME From d9b14360466df68a7313c27c8f182b48a25c6a84 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Thu, 23 Dec 2021 11:39:34 +0100 Subject: [PATCH 3/5] remove ES_HOST used to run test locally Signed-off-by: Tetiana Kravchenko --- deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml index c7bd222c730..485ecfcbcec 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml @@ -547,7 +547,7 @@ spec: - name: ES_PASSWORD value: "" - name: ES_HOST - value: "elastic-package-stack_elasticsearch_1" + value: "" - name: NODE_NAME valueFrom: fieldRef: From 7c00ce97aa300be3570a36719b0dc60787a22339 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Thu, 23 Dec 2021 12:54:06 +0100 Subject: [PATCH 4/5] set default values for container parser implicitly Signed-off-by: Tetiana Kravchenko --- deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml | 4 +--- .../elastic-agent-standalone-daemonset-configmap.yaml | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml index 485ecfcbcec..effc25095dd 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml @@ -238,9 +238,7 @@ data: type: logs prospector.scanner.symlinks: true parsers: - - container: - stream: all - format: auto + - container: ~ # - ndjson: # target: json # - multiline: diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml index 3c1306456de..efdd8fed066 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml @@ -238,9 +238,7 @@ data: type: logs prospector.scanner.symlinks: true parsers: - - container: - stream: all - format: auto + - container: ~ # - ndjson: # target: json # - multiline: From 00408231d09f24d7ace4172a3d0d1f2240eef96a Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Thu, 23 Dec 2021 13:05:49 +0100 Subject: [PATCH 5/5] remove skip_older as it is a default value anyway Signed-off-by: Tetiana Kravchenko --- deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml | 1 - .../elastic-agent-standalone-daemonset-configmap.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml index effc25095dd..9f58ec9c4f3 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml @@ -54,7 +54,6 @@ data: - event period: 10s add_metadata: true - skip_older: true - data_stream: dataset: kubernetes.state_container type: metrics diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml index efdd8fed066..0ad4f609883 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml @@ -54,7 +54,6 @@ data: - event period: 10s add_metadata: true - skip_older: true - data_stream: dataset: kubernetes.state_container type: metrics