diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 77906646291a..7edf170a1075 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -244,6 +244,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add ISO8601 timestamp support in syslog metricset. {issue}8716[8716] {pull}10736[10736] - Add more info to message logged when a duplicated symlink file is found {pull}10845[10845] - Add Netflow module to enrich flow events with geoip data. {pull}10877[10877] +- Add Filebeat kubernetes module. {issue}10812[10812] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index d1fe115e4ae9..f45be83d1573 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -27,6 +27,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -6521,6 +6522,288 @@ type: keyword Kubernetes container image +-- + +[[exported-fields-kubernetes]] +== Kubernetes fields + +Module for handling logs produced by Kubenetes pods + + + +[float] +== coredns fields + +Fields from coredns logs after normalization + + + +*`coredns.duration`*:: ++ +-- +type: double + +-- + +*`coredns.client`*:: ++ +-- +type: ip + +-- + +*`coredns.size`*:: ++ +-- +type: integer + +-- + +*`coredns.dnssec_ok`*:: ++ +-- +type: boolean + +-- + +*`coredns.proto`*:: ++ +-- +type: keyword + +-- + +*`coredns.qclass`*:: ++ +-- +type: keyword + +-- + +*`coredns.rsize`*:: ++ +-- +type: integer + +-- + +*`coredns.log_level`*:: ++ +-- +type: keyword + +-- + +*`coredns.id`*:: ++ +-- +type: keyword + +-- + +*`coredns.rcode`*:: ++ +-- +type: keyword + +-- + +*`coredns.port`*:: ++ +-- +type: integer + +-- + +*`coredns.bufsize`*:: ++ +-- +type: integer + +-- + +*`coredns.qtype`*:: ++ +-- +type: keyword + +-- + +*`coredns.rflags`*:: ++ +-- +type: keyword + +coredns response flags + + +-- + +*`coredns.qname`*:: ++ +-- +type: keyword + +coredns query name + + +-- + +*`coredns.message`*:: ++ +-- +type: text + +coredns original log message + + +-- + +[float] +== kubernetes fields + +Kubernetes meta fields + + + + +*`kubernetes.replicaset.name`*:: ++ +-- +type: keyword + +-- + +[float] +== envoy fields + +Fields from envoy proxy logs after normalization + + + +*`envoy.log_type`*:: ++ +-- +type: keyword + +Envoy log type, normally ACCESS + + +-- + +*`envoy.method`*:: ++ +-- +type: keyword + +-- + +*`envoy.path`*:: ++ +-- +type: keyword + +-- + +*`envoy.proto`*:: ++ +-- +type: keyword + +-- + +*`envoy.response_code`*:: ++ +-- +type: integer + +-- + +*`envoy.response_flags`*:: ++ +-- +type: keyword + +-- + +*`envoy.bytes_received`*:: ++ +-- +type: long + +-- + +*`envoy.bytes_sent`*:: ++ +-- +type: long + +-- + +*`envoy.duration`*:: ++ +-- +type: double + +-- + +*`envoy.upstream_service_time`*:: ++ +-- +type: double + +-- + +*`envoy.forwarded_for`*:: ++ +-- +type: keyword + +-- + +*`envoy.http_user_agent`*:: ++ +-- +type: keyword + +-- + +*`envoy.request_id`*:: ++ +-- +type: keyword + +-- + +*`envoy.authority`*:: ++ +-- +type: keyword + +-- + +*`envoy.upstream_host`*:: ++ +-- +type: ip + +-- + +*`envoy.upstream_port`*:: ++ +-- +type: integer + +-- + +*`envoy.message`*:: ++ +-- +type: text + +envoy original log message + + -- [[exported-fields-log]] diff --git a/filebeat/docs/modules/kubernetes.asciidoc b/filebeat/docs/modules/kubernetes.asciidoc new file mode 100644 index 000000000000..4fc169c5286b --- /dev/null +++ b/filebeat/docs/modules/kubernetes.asciidoc @@ -0,0 +1,35 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-kubernetes]] +[role="xpack"] + +:modulename: kubernetes +:has-dashboards: true + +== Kubernetes Module + +This is a filebeat module for Kubernetes. It ingests logs for Kubernetes pods and containers, specifically coredns and envoy. + +[float] +=== Compatibility + +This module has been developed against Kubernetes v1.13.x, but is expected to work +with other versions of Kubernetes. + +[float] +=== Example dashboard + +This module comes with a sample dashboard. + +[role="screenshot"] +image::./images/Filebeat-Kubernetes-Dashboard.png[] + + +[float] +=== Fields + +For a description of each field in the module, see the +<> section. + diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index ea3b3d74997e..20bc52f784f7 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -12,6 +12,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -40,6 +41,7 @@ include::modules/iis.asciidoc[] include::modules/iptables.asciidoc[] include::modules/kafka.asciidoc[] include::modules/kibana.asciidoc[] +include::modules/kubernetes.asciidoc[] include::modules/logstash.asciidoc[] include::modules/mongodb.asciidoc[] include::modules/mysql.asciidoc[] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 1d0217307c29..6fcac808e0fe 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -245,6 +245,18 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: +#------------------------------ Kubernetes Module ------------------------------ +- module: kubernetes + # All logs + coredns: + enabled: true + envoy: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + #------------------------------- Logstash Module ------------------------------- #- module: logstash # logs diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index c6332bde38f4..731b9b6af694 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -10,6 +10,7 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/x-pack/filebeat/input/netflow" _ "github.com/elastic/beats/x-pack/filebeat/module/iptables" + _ "github.com/elastic/beats/x-pack/filebeat/module/kubernetes" _ "github.com/elastic/beats/x-pack/filebeat/module/netflow" _ "github.com/elastic/beats/x-pack/filebeat/module/suricata" _ "github.com/elastic/beats/x-pack/filebeat/module/zeek" diff --git a/x-pack/filebeat/module/kubernetes/README.md b/x-pack/filebeat/module/kubernetes/README.md new file mode 100644 index 000000000000..f064a07fe218 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/README.md @@ -0,0 +1,46 @@ +# kubernetes module + +This module enables log ingestion for kubernetes, specifically envoy and coredns filesets. + +## Caveats + +* Module is to be considered _beta_. + +## Download and install Filebeat + +Grab the filebeat binary from elastic.co, and install it by following the instructions. + +## Enable kubernetes module by deploying the daemon set yaml file +``` +kubectl apply -f k8s-ingest.yaml +``` +### Note the following section in the yaml file + +``` +filebeat.autodiscover: + providers: + - type: kubernetes + hints.enabled: true + default.disable: true +``` + +This enables auto-discovery and hints for filebeat. When default.disable is set to true (default value false), it will disable log harvesting for the pod/container, unless it has specific annotations enabled. This gives users more granular control on kubernetes log ingestion. + +### Sample kubernetes deployment configuration with annotations, and disable set to false, which enables log harvesting for the pods +``` +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: ambassador +spec: + replicas: 3 + template: + metadata: + annotations: + "co.elastic.logs/module": "kubernetes" + "co.elastic.logs/fileset": "envoy" + "co.elastic.logs/disable": "false" + labels: + service: ambassador +``` + diff --git a/x-pack/filebeat/module/kubernetes/_meta/config.yml b/x-pack/filebeat/module/kubernetes/_meta/config.yml new file mode 100644 index 000000000000..d48c23bcd718 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/_meta/config.yml @@ -0,0 +1,10 @@ +- module: kubernetes + # All logs + coredns: + enabled: true + envoy: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/x-pack/filebeat/module/kubernetes/_meta/docs.asciidoc b/x-pack/filebeat/module/kubernetes/_meta/docs.asciidoc new file mode 100644 index 000000000000..0c583996fb09 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/_meta/docs.asciidoc @@ -0,0 +1,22 @@ +[role="xpack"] + +:modulename: kubernetes +:has-dashboards: true + +== Kubernetes Module + +This is a filebeat module for Kubernetes. It ingests logs for Kubernetes pods and containers, specifically coredns and envoy. + +[float] +=== Compatibility + +This module has been developed against Kubernetes v1.13.x, but is expected to work +with other versions of Kubernetes. + +[float] +=== Example dashboard + +This module comes with a sample dashboard. + +[role="screenshot"] +image::./images/Filebeat-Kubernetes-Dashboard.png[] diff --git a/x-pack/filebeat/module/kubernetes/_meta/fields.yml b/x-pack/filebeat/module/kubernetes/_meta/fields.yml new file mode 100644 index 000000000000..c829ad8915a4 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/_meta/fields.yml @@ -0,0 +1,134 @@ +- key: kubernetes + title: Kubernetes + description: > + Module for handling logs produced by Kubenetes pods + fields: + - name: coredns + type: group + description: > + Fields from coredns logs after normalization + fields: + - name: duration + type: double + + - name: client + type: ip + + - name: size + type: integer + + - name: dnssec_ok + type: boolean + + - name: proto + type: keyword + + - name: qclass + type: keyword + + - name: rsize + type: integer + + - name: log_level + type: keyword + + - name: id + type: keyword + + - name: rcode + type: keyword + + - name: port + type: integer + + - name: bufsize + type: integer + + - name: qtype + type: keyword + + - name: rflags + type: keyword + description: > + coredns response flags + + - name: qname + type: keyword + description: > + coredns query name + + - name: message + type: text + description: > + coredns original log message + + - name: kubernetes + type: group + description: > + Kubernetes meta fields + fields: + - name: replicaset + type: group + fields: + - name: name + type: keyword + + - name: envoy + type: group + description: > + Fields from envoy proxy logs after normalization + fields: + - name: log_type + type: keyword + description: > + Envoy log type, normally ACCESS + + - name: method + type: keyword + + - name: path + type: keyword + + - name: proto + type: keyword + + - name: response_code + type: integer + + - name: response_flags + type: keyword + + - name: bytes_received + type: long + + - name: bytes_sent + type: long + + - name: duration + type: double + + - name: upstream_service_time + type: double + + - name: forwarded_for + type: keyword + + - name: http_user_agent + type: keyword + + - name: request_id + type: keyword + + - name: authority + type: keyword + + - name: upstream_host + type: ip + + - name: upstream_port + type: integer + + - name: message + type: text + description: > + envoy original log message diff --git a/x-pack/filebeat/module/kubernetes/_meta/images/Filebeat-Kubernetes-Dashboard.png b/x-pack/filebeat/module/kubernetes/_meta/images/Filebeat-Kubernetes-Dashboard.png new file mode 100644 index 000000000000..53e721e5032e Binary files /dev/null and b/x-pack/filebeat/module/kubernetes/_meta/images/Filebeat-Kubernetes-Dashboard.png differ diff --git a/x-pack/filebeat/module/kubernetes/_meta/kibana/7/dashboard/Filebeat-Kubernetes-Overview.json b/x-pack/filebeat/module/kubernetes/_meta/kibana/7/dashboard/Filebeat-Kubernetes-Overview.json new file mode 100644 index 000000000000..c85c3d630b5b --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/_meta/kibana/7/dashboard/Filebeat-Kubernetes-Overview.json @@ -0,0 +1,418 @@ +{ + "objects": [ + { + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": {}, + "gridData": { + "h": 15, + "i": "1", + "w": 15, + "x": 0, + "y": 0 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "version": "7.1.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 15, + "i": "2", + "w": 16, + "x": 31, + "y": 0 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "version": "7.1.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 15, + "i": "3", + "w": 16, + "x": 15, + "y": 0 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "version": "7.1.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 14, + "i": "4", + "w": 47, + "x": 0, + "y": 15 + }, + "panelIndex": "4", + "panelRefName": "panel_3", + "version": "7.1.0-SNAPSHOT" + } + ], + "timeRestore": false, + "title": "Kubernetes [Filebeat Module]", + "version": 1 + }, + "id": "b2926b80-33dd-11e9-acca-2f1960780e7b", + "migrationVersion": { + "dashboard": "7.0.0" + }, + "references": [ + { + "id": "1d801b20-33db-11e9-acca-2f1960780e7b", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "48f80920-33db-11e9-acca-2f1960780e7b", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "a3170f90-33dc-11e9-acca-2f1960780e7b", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "1002aab0-33dd-11e9-acca-2f1960780e7b", + "name": "panel_3", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2019-02-19T00:31:28.824Z", + "version": "WzI1MCwxXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "coredns top domains [Filebeat kubernetes]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "coredns.qname", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, + "isDonut": true, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "coredns top domains [Filebeat kubernetes]", + "type": "pie" + } + }, + "id": "1d801b20-33db-11e9-acca-2f1960780e7b", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-02-19T00:12:59.729Z", + "version": "WzI0NCwxXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "envoy top sites [Filebeat kubernetes]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "envoy.authority", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, + "isDonut": true, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "envoy top sites [Filebeat kubernetes]", + "type": "pie" + } + }, + "id": "48f80920-33db-11e9-acca-2f1960780e7b", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-02-19T00:14:12.658Z", + "version": "WzI0NSwxXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "coredns rcode [Filebeat kubernetes]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "coredns.rcode", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "maxFontSize": 72, + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + }, + "minFontSize": 18, + "orientation": "single", + "scale": "linear", + "showLabel": true + }, + "title": "coredns rcode [Filebeat kubernetes]", + "type": "tagcloud" + } + }, + "id": "a3170f90-33dc-11e9-acca-2f1960780e7b", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-02-19T00:23:53.353Z", + "version": "WzI0NiwxXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Pod traffic time-series [Filebeat kubernetes]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "line_width": 1, + "metrics": [ + { + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "count" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "kubernetes.pod.name" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "Pod traffic time-series [Filebeat kubernetes]", + "type": "metrics" + } + }, + "id": "1002aab0-33dd-11e9-acca-2f1960780e7b", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2019-02-19T00:30:36.690Z", + "version": "WzI0OSwxXQ==" + } + ], + "version": "7.1.0-SNAPSHOT" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/kubernetes/coredns/config/coredns.yml b/x-pack/filebeat/module/kubernetes/coredns/config/coredns.yml new file mode 100644 index 000000000000..3328778d7b6e --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/coredns/config/coredns.yml @@ -0,0 +1,14 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +tags: {{.tags}} +json.keys_under_root: true +processors: + - dissect: + tokenizer: "%{timestamp} [%{log_level}] %{client}:%{port} - %{id} \"%{qtype} %{qclass} %{qname} %{proto} %{size} %{dnssec_ok} %{bufsize}\" %{rcode} %{rflags} %{rsize} %{duration}s" + field: "message" + target_prefix: "coredns" + - drop_fields: + fields: [time] diff --git a/x-pack/filebeat/module/kubernetes/coredns/ingest/pipeline.json b/x-pack/filebeat/module/kubernetes/coredns/ingest/pipeline.json new file mode 100644 index 000000000000..3ce0199799d5 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/coredns/ingest/pipeline.json @@ -0,0 +1,18 @@ +{ + "description": "Pipeline for normalizing Kubernetes coredns logs", + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = ctx['coredns']['timestamp']; ctx.coredns.remove('timestamp');", + "ignore_failure" : true + } + }, + { + "set": { + "field": "source.ip", + "value": "{{coredns.client}}" + } + } + ] +} diff --git a/x-pack/filebeat/module/kubernetes/coredns/manifest.yml b/x-pack/filebeat/module/kubernetes/coredns/manifest.yml new file mode 100644 index 000000000000..e201364e14a2 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/coredns/manifest.yml @@ -0,0 +1,14 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /var/lib/docker/containers/*/*-json.log + - name: tags + default: [coredns] + +ingest_pipeline: ingest/pipeline.json +input: config/coredns.yml + +requires.processors: +- name: geoip diff --git a/x-pack/filebeat/module/kubernetes/coredns/test/coredns-json.log b/x-pack/filebeat/module/kubernetes/coredns/test/coredns-json.log new file mode 100644 index 000000000000..145ed42678e5 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/coredns/test/coredns-json.log @@ -0,0 +1 @@ +{ "message": "2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \"A IN httpbin.org.cluster.local. udp 43 false 512\" NXDOMAIN qr,rd,ra 136 0.000102078s", "stream": "stdout", "time": "2019-02-12T00:27:28.903433597Z", "kubernetes": { "container": { "name": "coredns" }, "node": { "name": "minikube" }, "pod": { "uid": "d57d545e-2a9d-11e9-995f-08002730e0dc", "name": "coredns-86c58d9df4-jwhsg" }, "namespace": "kube-system", "replicaset": { "name": "coredns-86c58d9df4" }, "labels": { "pod-template-hash": "86c58d9df4", "k8s-app": "kube-dns" } } } diff --git a/x-pack/filebeat/module/kubernetes/coredns/test/coredns-json.log-expected.json b/x-pack/filebeat/module/kubernetes/coredns/test/coredns-json.log-expected.json new file mode 100644 index 000000000000..b5f352301e07 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/coredns/test/coredns-json.log-expected.json @@ -0,0 +1,41 @@ +[ + { + "@timestamp": "2019-02-12T00:27:28.903Z", + "coredns.bufsize": "512", + "coredns.client": "172.17.0.4", + "coredns.dnssec_ok": "false", + "coredns.duration": "0.000102078", + "coredns.id": "21583", + "coredns.log_level": "INFO", + "coredns.port": "36413", + "coredns.proto": "udp", + "coredns.qclass": "IN", + "coredns.qname": "httpbin.org.cluster.local.", + "coredns.qtype": "A", + "coredns.rcode": "NXDOMAIN", + "coredns.rflags": "qr,rd,ra", + "coredns.rsize": "136", + "coredns.size": "43", + "ecs.version": "1.0.0-beta2", + "event.dataset": "kubernetes.coredns", + "event.module": "kubernetes", + "fileset.name": "coredns", + "input.type": "log", + "kubernetes.container.name": "coredns", + "kubernetes.labels.k8s-app": "kube-dns", + "kubernetes.labels.pod-template-hash": "86c58d9df4", + "kubernetes.namespace": "kube-system", + "kubernetes.node.name": "minikube", + "kubernetes.pod.name": "coredns-86c58d9df4-jwhsg", + "kubernetes.pod.uid": "d57d545e-2a9d-11e9-995f-08002730e0dc", + "kubernetes.replicaset.name": "coredns-86c58d9df4", + "log.offset": 0, + "message": "2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \"A IN httpbin.org.cluster.local. udp 43 false 512\" NXDOMAIN qr,rd,ra 136 0.000102078s", + "service.type": "kubernetes", + "source.ip": "172.17.0.4", + "stream": "stdout", + "tags": [ + "coredns" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/kubernetes/envoy/config/envoy.yml b/x-pack/filebeat/module/kubernetes/envoy/config/envoy.yml new file mode 100644 index 000000000000..ad3fb8820d02 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/envoy/config/envoy.yml @@ -0,0 +1,17 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +include_lines: ['^ACCESS'] +json.keys_under_root: true +json.message_key: message +tags: {{.tags}} +processors: + - dissect: + tokenizer: "%{log_type} [%{timestamp}] \"%{method} %{path} %{proto}\" %{response_code} %{response_flags} %{bytes_received} %{bytes_sent} %{duration} %{upstream_service_time} \"%{forwarded_for}\" \"%{http_user_agent}\" \"%{request_id}\" \"%{authority}\" \"%{upstream_host}:%{upstream_port}\"" + field: "message" + target_prefix: "envoy" + - drop_fields: + fields: [time] + diff --git a/x-pack/filebeat/module/kubernetes/envoy/ingest/pipeline.json b/x-pack/filebeat/module/kubernetes/envoy/ingest/pipeline.json new file mode 100644 index 000000000000..ca4380df3e11 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/envoy/ingest/pipeline.json @@ -0,0 +1,25 @@ +{ + "description": "Pipeline for normalizing Kubernetes envoy logs", + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = ctx['envoy']['timestamp']; ctx.envoy.remove('timestamp');", + "ignore_failure" : true + } + }, + { + "set": { + "field": "user_agent.original", + "value": "{{envoy.http_user_agent}}", + "ignore_failure": true + } + }, + { + "user_agent": { + "field": "envoy.http_user_agent", + "ignore_missing": true + } + } + ] +} \ No newline at end of file diff --git a/x-pack/filebeat/module/kubernetes/envoy/manifest.yml b/x-pack/filebeat/module/kubernetes/envoy/manifest.yml new file mode 100644 index 000000000000..ae88ba780cd8 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/envoy/manifest.yml @@ -0,0 +1,11 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /var/lib/docker/containers/*/*-json.log + - name: tags + default: [envoy] + +ingest_pipeline: ingest/pipeline.json +input: config/envoy.yml diff --git a/x-pack/filebeat/module/kubernetes/envoy/test/envoy-json.log b/x-pack/filebeat/module/kubernetes/envoy/test/envoy-json.log new file mode 100644 index 000000000000..d58f89c1e05b --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/envoy/test/envoy-json.log @@ -0,0 +1,2 @@ +{"message":"ACCESS [2019-02-12T18:37:42.921Z] \"GET /httpbin/ip HTTP/1.1\" 200 - 0 43 217 216 \"172.17.0.5\" \"curl/7.59.0\" \"c62ec130-6dee-4f3b-a52b-4f0d1312d89d\" \"httpbin.org\" \"52.87.35.92:80\"\n","stream":"stdout","time":"2019-02-12T18:37:43.139620629Z", "kubernetes": { "container": { "name": "ambassador" }, "node": { "name": "minikube" }, "pod": { "uid": "e57d545e-2a9d-11e9-995f-08002730e0dc", "name": "ambassador-76c58d9df4-jwhsg" }, "namespace": "default", "labels": { "service": "ambassador" }}} + diff --git a/x-pack/filebeat/module/kubernetes/envoy/test/envoy-json.log-expected.json b/x-pack/filebeat/module/kubernetes/envoy/test/envoy-json.log-expected.json new file mode 100644 index 000000000000..8d6fd308ffab --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/envoy/test/envoy-json.log-expected.json @@ -0,0 +1,43 @@ +[ + { + "@timestamp": "2019-02-12T18:37:42.921Z", + "ecs.version": "1.0.0-beta2", + "envoy.authority": "httpbin.org", + "envoy.bytes_received": "0", + "envoy.bytes_sent": "43", + "envoy.duration": "217", + "envoy.forwarded_for": "172.17.0.5", + "envoy.http_user_agent": "curl/7.59.0", + "envoy.log_type": "ACCESS", + "envoy.method": "GET", + "envoy.path": "/httpbin/ip", + "envoy.proto": "HTTP/1.1", + "envoy.request_id": "c62ec130-6dee-4f3b-a52b-4f0d1312d89d", + "envoy.response_code": "200", + "envoy.response_flags": "-", + "envoy.upstream_host": "52.87.35.92", + "envoy.upstream_port": "80", + "envoy.upstream_service_time": "216", + "event.dataset": "kubernetes.envoy", + "event.module": "kubernetes", + "fileset.name": "envoy", + "input.type": "log", + "kubernetes.container.name": "ambassador", + "kubernetes.labels.service": "ambassador", + "kubernetes.namespace": "default", + "kubernetes.node.name": "minikube", + "kubernetes.pod.name": "ambassador-76c58d9df4-jwhsg", + "kubernetes.pod.uid": "e57d545e-2a9d-11e9-995f-08002730e0dc", + "log.offset": 0, + "message": "ACCESS [2019-02-12T18:37:42.921Z] \"GET /httpbin/ip HTTP/1.1\" 200 - 0 43 217 216 \"172.17.0.5\" \"curl/7.59.0\" \"c62ec130-6dee-4f3b-a52b-4f0d1312d89d\" \"httpbin.org\" \"52.87.35.92:80\"", + "service.type": "kubernetes", + "stream": "stdout", + "tags": [ + "envoy" + ], + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.59.0", + "user_agent.version": "7.59.0" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/kubernetes/fields.go b/x-pack/filebeat/module/kubernetes/fields.go new file mode 100644 index 000000000000..c92ffdd2bb78 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package kubernetes + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "kubernetes", asset.ModuleFieldsPri, AssetKubernetes); err != nil { + panic(err) + } +} + +// AssetKubernetes returns asset data. +// This is the base64 encoded gzipped contents of module/kubernetes. +func AssetKubernetes() string { + return "eJyclUFu4zAMRfc5BQ8wvYAXAwyKzmYwqx7AUCzaFiKLCkmldU8/sNPGUeokmqyC2I+fkqj//QQ7HCvYpS1yQEXZAKhTjxX8OX9mURp2UR2FCn5uAAD+kk0eoSWG3gTrXejAUycQmWxq0MJ2nEVmDYhkJ6HWobdSbQCeIJgBK2iI0QaZNXWMWEHHlOL8f6UtwO9ZAlqm4av42Ni0igyBeDDefZipai5Zei5dbeIF+GpsKW09bjKw8Q6DZpiLOSLuA3MgKHbIOWWDCDY17TJ0S+TRhByNTEoZtsPxjdhCzu0bb0TWwJzjshV66mqPB/T3FZ0t6NqQxftYJNb7a9umtmwT++ldweJab7r1ozs+W716cLpwjBIpCMJRJ1/C9PO49D4hj7NULjugiOlyYcV3LVMldp0Lxk9TPkmd2TCLgEInLhEBA6r5NNpVzzFG7xojmM97aZKXLYVn57ky1C8Kw4HGx2JkLp1s9z7+f5RMtrl66W7O5mVuOw1kKvrx2c+P8Ov5+eX19XL82lOB7aLRvoCaIgYKfPJ5z+tvZl513wm/bq/c1qOi1IwNugPmm/MUujVYLuP4O3gj3C8SNEVRRjPUgnxwDdbqLqy79k1oid8MW7R1S3x/k71qrJMg16a7XPyVQ98nFK1LUtYk7YmdjvfR02Z7EoWbn7QTWpbOj0XT0XWrwfQvAAD///r9ifI=" +} diff --git a/x-pack/filebeat/module/kubernetes/k8s-ingest.yaml b/x-pack/filebeat/module/kubernetes/k8s-ingest.yaml new file mode 100644 index 000000000000..7e54f88f68e8 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/k8s-ingest.yaml @@ -0,0 +1,142 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ingest-config + namespace: kube-system + labels: + k8s-app: ingest +data: + filebeat-k8s.yml: |- + logging.level: info + filebeat.config: + modules: + path: ${path.config}/modules.d/*.yml + + filebeat.autodiscover: + providers: + - type: kubernetes + hints.enabled: true + default.disable: true + + processors: + - add_kubernetes_metadata: + in_cluster: true + + output.elasticsearch: + ilm.enabled: false + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} + + setup.kibana: + host: '${KIBANA_HOST:kibana}:${KIBANA_PORT:5601}' +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: ingest + namespace: kube-system + labels: + k8s-app: ingest +spec: + template: + metadata: + labels: + k8s-app: ingest + spec: + serviceAccountName: ingest + terminationGracePeriodSeconds: 30 + containers: + - name: ingest + image: docker.elastic.co/beats/filebeat:8.0.0 + args: [ + "sh", "-c", "filebeat setup -e --modules kubernetes -c /etc/filebeat-k8s.yml && filebeat -e -c /etc/filebeat-k8s.yml" + ] + env: + - name: ELASTICSEARCH_HOST + value: 192.168.99.1 + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: KIBANA_HOST + value: 192.168.99.1 + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + resources: + limits: + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + volumeMounts: + - name: config + mountPath: /etc/filebeat-k8s.yml + readOnly: true + subPath: filebeat-k8s.yml + - name: data + mountPath: /usr/share/filebeat/data + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + #- name: varlogcontainers + # mountPath: /var/log/containers + # readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0600 + name: ingest-config + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart + - name: data + hostPath: + path: /var/lib/filebeat-data + type: DirectoryOrCreate +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: ingest +subjects: +- kind: ServiceAccount + name: ingest + namespace: kube-system +roleRef: + kind: ClusterRole + name: ingest + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: ingest + labels: + k8s-app: ingest +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + verbs: + - get + - watch + - list +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingest + namespace: kube-system + labels: + k8s-app: ingest +--- + diff --git a/x-pack/filebeat/module/kubernetes/module.yml b/x-pack/filebeat/module/kubernetes/module.yml new file mode 100644 index 000000000000..48a3fc50d162 --- /dev/null +++ b/x-pack/filebeat/module/kubernetes/module.yml @@ -0,0 +1,3 @@ +dashboards: +- id: b2926b80-33dd-11e9-acca-2f1960780e7b + file: Filebeat-Kubernetes-Overview.json diff --git a/x-pack/filebeat/modules.d/kubernetes.yml.disabled b/x-pack/filebeat/modules.d/kubernetes.yml.disabled new file mode 100644 index 000000000000..c200bb4be479 --- /dev/null +++ b/x-pack/filebeat/modules.d/kubernetes.yml.disabled @@ -0,0 +1,13 @@ +# Module: kubernetes +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-kubernetes.html + +- module: kubernetes + # All logs + coredns: + enabled: true + envoy: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: