diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7bdff1ed2ab..b952c9dc0c4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -309,6 +309,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for loading custom NetFlow and IPFIX field definitions to netflow input. {pull}10945[10945] - Added categorization fields for SSH login events in the system/auth fileset. {pull}11334[11334] - Add support for MySQL 8.0 slow logs and tests also for Percona 8.0 and MariaDB 10.3. {pull}11417[11417] +- New Filebeat coredns module to ingest coredns logs. It supports both native coredns deployment and coredns deployment in kubernetes. {pull}11200[11200] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 72fbb9982b2..d4b5e29fcea 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -16,6 +16,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -1125,6 +1126,114 @@ type: alias alias to: cloud.region +-- + +[[exported-fields-coredns]] +== Coredns fields + +Module for handling logs produced by coredns + + + +[float] +== coredns fields + +coredns fields after normalization + + + +*`coredns.id`*:: ++ +-- +type: keyword + +id of the DNS transaction + + +-- + +*`coredns.query.size`*:: ++ +-- +type: integer + +format: bytes + +size of the DNS query + + +-- + +*`coredns.query.class`*:: ++ +-- +type: keyword + +DNS query class + + +-- + +*`coredns.query.name`*:: ++ +-- +type: keyword + +DNS query name + + +-- + +*`coredns.query.type`*:: ++ +-- +type: keyword + +DNS query type + + +-- + +*`coredns.response.code`*:: ++ +-- +type: keyword + +DNS response code + + +-- + +*`coredns.response.flags`*:: ++ +-- +type: keyword + +DNS response flags + + +-- + +*`coredns.response.size`*:: ++ +-- +type: integer + +format: bytes + +size of the DNS response + + +-- + +*`coredns.dnssec_ok`*:: ++ +-- +type: boolean + +dnssec flag + + -- [[exported-fields-docker-processor]] diff --git a/filebeat/docs/modules/coredns.asciidoc b/filebeat/docs/modules/coredns.asciidoc new file mode 100644 index 00000000000..6e6d9040e7b --- /dev/null +++ b/filebeat/docs/modules/coredns.asciidoc @@ -0,0 +1,37 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-coredns]] +[role="xpack"] + +:modulename: coredns +:has-dashboards: true + +== Coredns Module + +This is a filebeat module for coredns. It supports both standalone coredns deployment and +coredns deployment in Kubernetes. Standalone coredns deployment uses the log fileset, while +Kubernetes coredns deployment uses the kubernetes fileset. + +[float] +=== Compatibility + +Although this module has been developed against Kubernetes v1.13.x, it is expected to work +with other versions of Kubernetes. + +[float] +=== Example dashboard + +This module comes with a sample dashboard. + +[role="screenshot"] +image::./images/Coredns_Overview_Dashboard.jpg[] + + +[float] +=== Fields + +For a description of each field in the module, see the +<> section. + diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index ea3b3d74997..f7e808fddab 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -5,6 +5,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -33,6 +34,7 @@ This file is generated! See scripts/docs_collector.py include::modules-overview.asciidoc[] include::modules/apache.asciidoc[] include::modules/auditd.asciidoc[] +include::modules/coredns.asciidoc[] include::modules/elasticsearch.asciidoc[] include::modules/haproxy.asciidoc[] include::modules/icinga.asciidoc[] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index dc315b4f67c..28cafb25cd2 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -82,6 +82,16 @@ filebeat.modules: # can be added under this section. #input: +#------------------------------- Coredns Module ------------------------------- +- module: coredns + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + #---------------------------- Elasticsearch Module ---------------------------- - module: elasticsearch # Server log diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index c6332bde38f..b3df5fd1519 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -9,6 +9,7 @@ package include import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/x-pack/filebeat/input/netflow" + _ "github.com/elastic/beats/x-pack/filebeat/module/coredns" _ "github.com/elastic/beats/x-pack/filebeat/module/iptables" _ "github.com/elastic/beats/x-pack/filebeat/module/netflow" _ "github.com/elastic/beats/x-pack/filebeat/module/suricata" diff --git a/x-pack/filebeat/module/coredns/README.md b/x-pack/filebeat/module/coredns/README.md new file mode 100644 index 00000000000..8aeaad3a211 --- /dev/null +++ b/x-pack/filebeat/module/coredns/README.md @@ -0,0 +1,160 @@ +# Coredns Module + +This is a filebeat module for coredns. It supports both standalone coredns deployment and +coredns deployment in Kubernetes. + +## Caveats + +* Module is to be considered _beta_. + +## Download and install Filebeat + +Grab the filebeat binary from elastic.co, and install it by following the instructions. + +## Deployment Scenario #1: coredns native deployment + +Make sure to update coredns configuration to enable log plugin. This module assumes that coredns log +entries will be written to /var/log/coredns.log. Should it be not the case, please point the module +log path to the path of the log file. + +Update filebeat.yml to point to Elasticsearch and Kibana. +Setup Filebeat. +``` +./filebeat setup --modules coredns -e +``` + +Enable the Filebeat coredns module +``` +./filebeat modules enable coredns +``` + +Start Filebeat +``` +./filebeat -e +``` + +Now, the Coredns logs and dashboard should appear in Kibana. + + +## Deployment Scenario #2: coredns for kubernetes + +For Kubernetes deployment, the filebeat daemon-set yaml file needs to be deployed to the +Kubernetes cluster. Sample configuration files is provided under the `beats/deploy/filebeat` +directory, and can be deployed by doing the following: +``` +kubectl apply -f filebeat +``` + +#### Note the following section in the ConfigMap, make changes to the yaml file if necessary +``` + filebeat.autodiscover: + providers: + - type: kubernetes + hints.enabled: true + default.disable: true + + processors: + - add_kubernetes_metadata: + in_cluster: true +``` + +This enables auto-discovery and hints for filebeat. When default.disable is set to true (default value is false), it will disable log harvesting for the pod/container, unless it has specific annotations enabled. This gives users more granular control on kubernetes log ingestion. The `add_kubernetes_metadata` processor will add enrichment data for Kubernetes to the ingest logs. + +#### Note the following section in the DaemonSet, make changes to the yaml file if necessary +``` +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +spec: + template: + metadata: + labels: + k8s-app: filebeat + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + containers: + - name: filebeat + image: docker.elastic.co/beats/filebeat:%VERSION% + args: [ + "sh", "-c", "filebeat setup -e --modules coredns -c /etc/filebeat.yml && filebeat -e -c /etc/filebeat.yml" + ] + env: + # Edit the following values to reflect your setup accordingly + - name: ELASTICSEARCH_HOST + value: 192.168.99.1 + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: KIBANA_HOST + value: 192.168.99.1 +``` + +The module setup step can also be done separately without Kubernetes if applicable, and in that case, the args can be simplified to: +``` + args: [ + "sh", "-c", "filebeat -e -c /etc/filebeat.yml" + ] +``` + +### Note that you probably need to update the coredns configmap to enable logging, and coredns deployment to add proper annotations. + +##### Sample ConfigMap for coredns: + +``` +apiVersion: v1 +data: + Corefile: | + .:53 { + log + errors + health + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods verified + endpoint_pod_names + upstream + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + proxy . /etc/resolv.conf + cache 30 + loop + reload + loadbalance + } +kind: ConfigMap +metadata: + creationTimestamp: "2019-01-31T21:02:57Z" + name: coredns + namespace: kube-system + resourceVersion: "185717" + selfLink: /api/v1/namespaces/kube-system/configmaps/coredns + uid: 95a5d5cb-259b-11e9-8e5d-080027971f3c +``` + +#### Sample Deployment for coredns. Note the annotations. + +``` +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: coredns +spec: + replicas: 2 + template: + metadata: + annotations: + "co.elastic.logs/module": "coredns" + "co.elastic.logs/fileset": "log" + "co.elastic.logs/disable": "false" + labels: + k8s-app: coredns + spec: + +``` + diff --git a/x-pack/filebeat/module/coredns/_meta/config.yml b/x-pack/filebeat/module/coredns/_meta/config.yml new file mode 100644 index 00000000000..d9ef777bde5 --- /dev/null +++ b/x-pack/filebeat/module/coredns/_meta/config.yml @@ -0,0 +1,8 @@ +- module: coredns + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/x-pack/filebeat/module/coredns/_meta/docs.asciidoc b/x-pack/filebeat/module/coredns/_meta/docs.asciidoc new file mode 100644 index 00000000000..613b05bca4c --- /dev/null +++ b/x-pack/filebeat/module/coredns/_meta/docs.asciidoc @@ -0,0 +1,24 @@ +[role="xpack"] + +:modulename: coredns +:has-dashboards: true + +== Coredns Module + +This is a filebeat module for coredns. It supports both standalone coredns deployment and +coredns deployment in Kubernetes. Standalone coredns deployment uses the log fileset, while +Kubernetes coredns deployment uses the kubernetes fileset. + +[float] +=== Compatibility + +Although this module has been developed against Kubernetes v1.13.x, it is expected to work +with other versions of Kubernetes. + +[float] +=== Example dashboard + +This module comes with a sample dashboard. + +[role="screenshot"] +image::./images/Coredns_Overview_Dashboard.jpg[] diff --git a/x-pack/filebeat/module/coredns/_meta/fields.yml b/x-pack/filebeat/module/coredns/_meta/fields.yml new file mode 100644 index 00000000000..7bc11ae1470 --- /dev/null +++ b/x-pack/filebeat/module/coredns/_meta/fields.yml @@ -0,0 +1,57 @@ +- key: coredns + title: Coredns + description: > + Module for handling logs produced by coredns + fields: + - name: coredns + type: group + description: > + coredns fields after normalization + fields: + - name: id + type: keyword + description: > + id of the DNS transaction + + - name: query.size + type: integer + format: bytes + description: > + size of the DNS query + + - name: query.class + type: keyword + description: > + DNS query class + + - name: query.name + type: keyword + description: > + DNS query name + + - name: query.type + type: keyword + description: > + DNS query type + + - name: response.code + type: keyword + description: > + DNS response code + + - name: response.flags + type: keyword + description: > + DNS response flags + + - name: response.size + type: integer + format: bytes + description: > + size of the DNS response + + - name: dnssec_ok + type: boolean + description: > + dnssec flag + \ No newline at end of file diff --git a/x-pack/filebeat/module/coredns/_meta/images/Coredns_Overview_Dashboard.jpg b/x-pack/filebeat/module/coredns/_meta/images/Coredns_Overview_Dashboard.jpg new file mode 100644 index 00000000000..70921fa9baf Binary files /dev/null and b/x-pack/filebeat/module/coredns/_meta/images/Coredns_Overview_Dashboard.jpg differ diff --git a/x-pack/filebeat/module/coredns/_meta/kibana/8/dashboard/Coredns-Overview-Dashboard.json b/x-pack/filebeat/module/coredns/_meta/kibana/8/dashboard/Coredns-Overview-Dashboard.json new file mode 100644 index 00000000000..d72e943e141 --- /dev/null +++ b/x-pack/filebeat/module/coredns/_meta/kibana/8/dashboard/Coredns-Overview-Dashboard.json @@ -0,0 +1,464 @@ +{ + "objects": [ + { + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": {}, + "gridData": { + "h": 14, + "i": "1", + "w": 41, + "x": 4, + "y": 7 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 7, + "i": "2", + "w": 20, + "x": 4, + "y": 0 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 7, + "i": "3", + "w": 21, + "x": 24, + "y": 0 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 6, + "i": "4", + "w": 41, + "x": 4, + "y": 21 + }, + "panelIndex": "4", + "panelRefName": "panel_3", + "version": "8.0.0-SNAPSHOT" + } + ], + "timeRestore": false, + "title": "Coredns Overview Dashboard", + "version": 1 + }, + "id": "53aa1f70-443e-11e9-8548-ab7fbe04f038", + "migrationVersion": { + "dashboard": "7.0.0" + }, + "references": [ + { + "id": "3ad75810-4429-11e9-8548-ab7fbe04f038", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "9dc640e0-4432-11e9-8548-ab7fbe04f038", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "75743f70-443c-11e9-8548-ab7fbe04f038", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "cfde7fb0-443d-11e9-8548-ab7fbe04f038", + "name": "panel_3", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2019-03-11T20:43:54.420Z", + "version": "WzE0ODgsM10=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Top Domains [Coredns]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "coredns.query.name", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "maxFontSize": 72, + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + }, + "minFontSize": 18, + "orientation": "single", + "scale": "linear", + "showLabel": true + }, + "title": "Top Domains [Coredns]", + "type": "tagcloud" + } + }, + "id": "3ad75810-4429-11e9-8548-ab7fbe04f038", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "8d890080-413c-11e9-8548-ab7fbe04f038", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-03-11T18:12:28.303Z", + "version": "WzE0ODMsM10=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Total DNS Queries [Coredns]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + } + ], + "params": { + "addLegend": false, + "addTooltip": true, + "metric": { + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 10000 + } + ], + "invertColors": false, + "labels": { + "show": true + }, + "metricColorMode": "None", + "percentageMode": false, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": 60, + "labelColor": false, + "subText": "" + }, + "useRanges": false + }, + "type": "metric" + }, + "title": "Total DNS Queries [Coredns]", + "type": "metric" + } + }, + "id": "9dc640e0-4432-11e9-8548-ab7fbe04f038", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "8d890080-413c-11e9-8548-ab7fbe04f038", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-03-11T19:19:39.757Z", + "version": "WzE0ODQsM10=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Unique Domains [Coredns]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Unique Domains", + "field": "coredns.query.name" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "params": { + "addLegend": false, + "addTooltip": true, + "metric": { + "bucket": { + "accessor": 0, + "aggType": "terms", + "format": { + "id": "terms", + "params": { + "id": "string", + "missingBucketLabel": "Missing", + "otherBucketLabel": "Other" + } + }, + "params": {} + }, + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 10000 + } + ], + "invertColors": false, + "labels": { + "show": true + }, + "metricColorMode": "None", + "metrics": [ + { + "accessor": 0, + "aggType": "cardinality", + "format": { + "id": "number" + }, + "params": {} + } + ], + "percentageMode": false, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": 60, + "labelColor": false, + "subText": "" + }, + "useRanges": false + }, + "type": "metric" + }, + "title": "Unique Domains [Coredns]", + "type": "metric" + } + }, + "id": "75743f70-443c-11e9-8548-ab7fbe04f038", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "8d890080-413c-11e9-8548-ab7fbe04f038", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-03-12T01:26:19.218Z", + "version": "WzE0OTMsM10=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "tags:\"coredns\"" + } + } + }, + "title": "Time Series Visualizer [Coredns]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "65ad37b0-443f-11e9-94ba-69b05a5f82b8" + } + ], + "bar_color_rules": [ + { + "id": "e1f6cda0-443e-11e9-94ba-69b05a5f82b8" + } + ], + "default_index_pattern": "filebeat-*", + "gauge_color_rules": [ + { + "id": "6996a6e0-443f-11e9-94ba-69b05a5f82b8" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": "fileset.name:kubernetes", + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "CoreDNS Kubernetes", + "line_width": 1, + "metrics": [ + { + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "count" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "filter", + "stacked": "none", + "terms_field": "fileset.name" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": "fileset.name:log", + "formatter": "number", + "id": "3c8999f0-443f-11e9-94ba-69b05a5f82b8", + "label": "CoreDNS Native", + "line_width": 1, + "metrics": [ + { + "id": "3c8999f1-443f-11e9-94ba-69b05a5f82b8", + "type": "count" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "filter", + "stacked": "none" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "Time Series Visualizer [Coredns]", + "type": "metrics" + } + }, + "id": "cfde7fb0-443d-11e9-8548-ab7fbe04f038", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2019-03-11T20:51:52.103Z", + "version": "WzE0ODksM10=" + } + ], + "version": "8.0.0-SNAPSHOT" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/coredns/fields.go b/x-pack/filebeat/module/coredns/fields.go new file mode 100644 index 00000000000..2275e8e05c2 --- /dev/null +++ b/x-pack/filebeat/module/coredns/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package coredns + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "coredns", asset.ModuleFieldsPri, AssetCoredns); err != nil { + panic(err) + } +} + +// AssetCoredns returns asset data. +// This is the base64 encoded gzipped contents of module/coredns. +func AssetCoredns() string { + return "eJy00kFugzAQheE9p3gXSA7Aopt22256gMqxB2LF8dAZo4qcvsIEApIVqYm6S0ae78eIHU401LAs5KJWQPIpUI3XZeBIrfgueY41XioAeGfXB0LDgqOJLvjYInCr6IRdb8nhMKzIxlNwWlfADtGcaZ0D0tBRjVa47/L/Qg/zwpWCaRIJIsvZBH8x49l88Fa6tby7GlPoRMMPyzwrxgDvwA3SkfD28YkkJqqxubKhv3uSYa/+QpuEj4lakuusGZ8y1TgMifR+dpTW4eyXkjYY1b9fazExAQV5/P0MnPcL7og94+b9jSukHUelvWX3ID0TyERZb4JpH3zVCz8ZZf//vp45kQ9s4i6qkv3i0yZ8YA5k4v3ItJpvtMyq3wAAAP//8MJAFA==" +} diff --git a/x-pack/filebeat/module/coredns/log/config/coredns.yml b/x-pack/filebeat/module/coredns/log/config/coredns.yml new file mode 100644 index 00000000000..c085c9e3aab --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/config/coredns.yml @@ -0,0 +1,7 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +tags: {{.tags}} +processors: diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.json b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.json new file mode 100644 index 00000000000..bea5c11f4ed --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.json @@ -0,0 +1,101 @@ +{ + "description": "Pipeline for normalizing Kubernetes coredns logs", + "processors": [ + { + "pipeline": { + "if": "ctx.message.charAt(0) == (char)(\"{\")", + "name": "{< IngestPipeline "pipeline-json" >}" + } + }, + { + "pipeline": { + "if": "ctx.message.charAt(0) != (char)(\"{\")", + "name": "{< IngestPipeline "pipeline-plaintext" >}" + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = ctx['timestamp']; ctx.remove('timestamp');", + "ignore_failure" : true + } + }, + { + "script": { + "lang": "painless", + "source": "ctx['source'] = new HashMap(); if (ctx.temp.source.charAt(0) == (char)(\"[\")) { def p = ctx.temp.source.indexOf (']'); def l = ctx.temp.source.length(); ctx.source.address = ctx.temp.source.substring(1, p); ctx.source.port = ctx.temp.source.substring(p+2, l);} else { def p = ctx.temp.source.indexOf (':'); def l = ctx.temp.source.length(); ctx.source.address = ctx.temp.source.substring(0, p); ctx.source.port = ctx.temp.source.substring(p+1, l);} ctx.remove('temp');", + "if": "ctx.temp?.source != null" + } + }, + { + "set": { + "field": "source.ip", + "value": "{{source.address}}", + "if": "ctx.source?.address != null" + } + }, + { + "convert" : { + "field" : "source.port", + "type": "integer" + } + }, + { + "convert" : { + "field" : "coredns.duration", + "type": "double" + } + }, + { + "convert" : { + "field" : "coredns.query.size", + "type": "long" + } + }, + { + "convert" : { + "field" : "coredns.response.size", + "type": "long" + } + }, + { + "convert" : { + "field" : "coredns.dnssec_ok", + "type": "boolean" + } + }, + { + "uppercase": { + "field": "coredns.response.flags" + } + }, + { + "split": { + "field": "coredns.response.flags", + "separator": "," + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = Math.round(ctx.coredns.duration * params.scale)", + "params": { + "scale": 1000000000 + }, + "if": "ctx.coredns?.duration != null" + } + }, + { + "remove": { + "field": "coredns.duration", + "ignore_missing": true + } + } + ], + "on_failure" : [{ + "set" : { + "field" : "error.message", + "value" : "{{ _ingest.on_failure_message }}" + } + }] +} diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.json b/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.json new file mode 100644 index 00000000000..40364fa0b6d --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.json @@ -0,0 +1,42 @@ +{ + "description": "Pipeline for dissecting message field in JSON logs", + "processors": [ + { + "json" : { + "field" : "message", + "target_field" : "json" + } + }, + { + "dissect": { + "field": "json.message", + "pattern": "%{timestamp} [%{log.level}] %{temp.source} - %{coredns.id} \"%{coredns.query.type} %{coredns.query.class} %{coredns.query.name} %{network.transport} %{coredns.query.size} %{coredns.dnssec_ok} %{bufsize}\" %{coredns.response.code} %{coredns.response.flags} %{coredns.response.size} %{coredns.duration}s" + } + }, + { + "remove": { + "field": ["message"], + "ignore_failure" : true + } + }, + { + "rename": { + "field": "json.message", + "target_field": "message", + "ignore_failure" : true + } + }, + { + "remove": { + "field": ["json", "bufsize"], + "ignore_failure" : true + } + } + ], + "on_failure" : [{ + "set" : { + "field" : "error.message", + "value" : "{{ _ingest.on_failure_message }}" + } + }] +} diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.json b/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.json new file mode 100644 index 00000000000..f1b0c1d26ea --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.json @@ -0,0 +1,23 @@ +{ + "description": "Pipeline for dissecting text in plaintext logs", + "processors": [ + { + "dissect": { + "field": "message", + "pattern": "%{timestamp} [%{log.level}] %{temp.source} - %{coredns.id} \"%{coredns.query.type} %{coredns.query.class} %{coredns.query.name} %{network.transport} %{coredns.query.size} %{coredns.dnssec_ok} %{bufsize}\" %{coredns.response.code} %{coredns.response.flags} %{coredns.response.size} %{coredns.duration}s" + } + }, + { + "remove": { + "field": ["bufsize"], + "ignore_failure" : true + } + } + ], + "on_failure" : [{ + "set" : { + "field" : "error.message", + "value" : "{{ _ingest.on_failure_message }}" + } + }] +} diff --git a/x-pack/filebeat/module/coredns/log/manifest.yml b/x-pack/filebeat/module/coredns/log/manifest.yml new file mode 100644 index 00000000000..34997379524 --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/manifest.yml @@ -0,0 +1,15 @@ +module_version: 1.0 + +var: + - name: paths + default: + # Change this if it is not "/var/log/coredns.log" in native deployment mode + - /var/log/coredns.log + - name: tags + default: [coredns] + +ingest_pipeline: + - ingest/pipeline-entry.json + - ingest/pipeline-json.json + - ingest/pipeline-plaintext.json +input: config/coredns.yml diff --git a/x-pack/filebeat/module/coredns/log/test/coredns-json.log b/x-pack/filebeat/module/coredns/log/test/coredns-json.log new file mode 100644 index 00000000000..9a2f9b6dea4 --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/test/coredns-json.log @@ -0,0 +1,3 @@ +{"message":"2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \"A IN httpbin.org.cluster.local. udp 43 false 512\" NXDOMAIN qr,rd,ra 136 0.000102078s", "stream": "stdout", "time": "2019-02-12T00:27:28.903433597Z", "kubernetes": { "container": { "name": "coredns" }, "node": { "name": "minikube" }, "pod": { "uid": "d57d545e-2a9d-11e9-995f-08002730e0dc", "name": "coredns-86c58d9df4-jwhsg" }, "namespace": "kube-system", "replicaset": { "name": "coredns-86c58d9df4" }, "labels": { "pod-template-hash": "86c58d9df4", "k8s-app": "kube-dns" } } } +{"message":"2019-03-19T02:57:23.213Z [INFO] 172.17.0.9:37723 - 6966 \"A IN httpbin.org. udp 29 false 512\" NOERROR qr,rd,ra 83 0.000082083s\n","stream":"stdout","time":"2019-03-19T02:57:23.214583742Z", "kubernetes": { "container": { "name": "coredns" }, "node": { "name": "minikube" }, "pod": { "uid": "d57d545e-2a9d-11e9-995f-08002730e0dc", "name": "coredns-86c58d9df4-jwhsg" }, "namespace": "kube-system", "replicaset": { "name": "coredns-86c58d9df4" }, "labels": { "pod-template-hash": "86c58d9df4", "k8s-app": "kube-dns" } } } +{"message":"2019-03-11T07:16:34.013Z [INFO] [::1]:37915 - 62762 \"AAAA IN czbaoyu.com. udp 29 false 512\" NOERROR qr,rd,ra 100 0.00006286s\n","stream":"stdout","time":"2019-03-11T07:16:34.013970788Z", "kubernetes": { "container": { "name": "coredns" }, "node": { "name": "minikube" }, "pod": { "uid": "d57d545e-2a9d-11e9-995f-08002730e0dc", "name": "coredns-86c58d9df4-jwhsg" }, "namespace": "kube-system", "replicaset": { "name": "coredns-86c58d9df4" }, "labels": { "pod-template-hash": "86c58d9df4", "k8s-app": "kube-dns" } } } diff --git a/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json new file mode 100644 index 00000000000..3764287da3c --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json @@ -0,0 +1,101 @@ +[ + { + "@timestamp": "2019-02-12T00:27:28.903Z", + "coredns.dnssec_ok": false, + "coredns.id": "21583", + "coredns.query.class": "IN", + "coredns.query.name": "httpbin.org.cluster.local.", + "coredns.query.size": 43, + "coredns.query.type": "A", + "coredns.response.code": "NXDOMAIN", + "coredns.response.flags": [ + "QR", + "RD", + "RA" + ], + "coredns.response.size": 136, + "ecs.version": "1.0.0", + "event.dataset": "coredns.log", + "event.duration": 102078, + "event.module": "coredns", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.offset": 0, + "message": "2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \"A IN httpbin.org.cluster.local. udp 43 false 512\" NXDOMAIN qr,rd,ra 136 0.000102078s", + "network.transport": "udp", + "service.type": "coredns", + "source.address": "172.17.0.4", + "source.ip": "172.17.0.4", + "source.port": 36413, + "tags": [ + "coredns" + ] + }, + { + "@timestamp": "2019-03-19T02:57:23.213Z", + "coredns.dnssec_ok": false, + "coredns.id": "6966", + "coredns.query.class": "IN", + "coredns.query.name": "httpbin.org.", + "coredns.query.size": 29, + "coredns.query.type": "A", + "coredns.response.code": "NOERROR", + "coredns.response.flags": [ + "QR", + "RD", + "RA" + ], + "coredns.response.size": 83, + "ecs.version": "1.0.0", + "event.dataset": "coredns.log", + "event.duration": 82083, + "event.module": "coredns", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.offset": 550, + "message": "2019-03-19T02:57:23.213Z [INFO] 172.17.0.9:37723 - 6966 \"A IN httpbin.org. udp 29 false 512\" NOERROR qr,rd,ra 83 0.000082083s\n", + "network.transport": "udp", + "service.type": "coredns", + "source.address": "172.17.0.9", + "source.ip": "172.17.0.9", + "source.port": 37723, + "tags": [ + "coredns" + ] + }, + { + "@timestamp": "2019-03-11T07:16:34.013Z", + "coredns.dnssec_ok": false, + "coredns.id": "62762", + "coredns.query.class": "IN", + "coredns.query.name": "czbaoyu.com.", + "coredns.query.size": 29, + "coredns.query.type": "AAAA", + "coredns.response.code": "NOERROR", + "coredns.response.flags": [ + "QR", + "RD", + "RA" + ], + "coredns.response.size": 100, + "ecs.version": "1.0.0", + "event.dataset": "coredns.log", + "event.duration": 62860, + "event.module": "coredns", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.offset": 1081, + "message": "2019-03-11T07:16:34.013Z [INFO] [::1]:37915 - 62762 \"AAAA IN czbaoyu.com. udp 29 false 512\" NOERROR qr,rd,ra 100 0.00006286s\n", + "network.transport": "udp", + "service.type": "coredns", + "source.address": "::1", + "source.ip": "::1", + "source.port": 37915, + "tags": [ + "coredns" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/coredns/log/test/coredns.log b/x-pack/filebeat/module/coredns/log/test/coredns.log new file mode 100644 index 00000000000..ede11815be5 --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/test/coredns.log @@ -0,0 +1,2 @@ +2019-03-06T08:55:28.903Z [INFO] 172.17.0.4:36413 - 21583 "A IN httpbin.org.cluster.local. udp 43 false 512" NXDOMAIN qr,rd,ra 136 0.000102078s +2019-03-18T22:13:36.289-07:00 [INFO] [::1]:57413 - 14639 "A IN www.yahoo.com. udp 42 false 4096" NOERROR qr,rd,ra 188 0.020948545s diff --git a/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json new file mode 100644 index 00000000000..2b86692ec8e --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json @@ -0,0 +1,68 @@ +[ + { + "@timestamp": "2019-03-06T08:55:28.903Z", + "coredns.dnssec_ok": false, + "coredns.id": "21583", + "coredns.query.class": "IN", + "coredns.query.name": "httpbin.org.cluster.local.", + "coredns.query.size": 43, + "coredns.query.type": "A", + "coredns.response.code": "NXDOMAIN", + "coredns.response.flags": [ + "QR", + "RD", + "RA" + ], + "coredns.response.size": 136, + "ecs.version": "1.0.0", + "event.dataset": "coredns.log", + "event.duration": 102078, + "event.module": "coredns", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.offset": 0, + "message": "2019-03-06T08:55:28.903Z [INFO] 172.17.0.4:36413 - 21583 \"A IN httpbin.org.cluster.local. udp 43 false 512\" NXDOMAIN qr,rd,ra 136 0.000102078s", + "network.transport": "udp", + "service.type": "coredns", + "source.address": "172.17.0.4", + "source.ip": "172.17.0.4", + "source.port": 36413, + "tags": [ + "coredns" + ] + }, + { + "@timestamp": "2019-03-18T22:13:36.289-07:00", + "coredns.dnssec_ok": false, + "coredns.id": "14639", + "coredns.query.class": "IN", + "coredns.query.name": "www.yahoo.com.", + "coredns.query.size": 42, + "coredns.query.type": "A", + "coredns.response.code": "NOERROR", + "coredns.response.flags": [ + "QR", + "RD", + "RA" + ], + "coredns.response.size": 188, + "ecs.version": "1.0.0", + "event.dataset": "coredns.log", + "event.duration": 20948545, + "event.module": "coredns", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.offset": 143, + "message": "2019-03-18T22:13:36.289-07:00 [INFO] [::1]:57413 - 14639 \"A IN www.yahoo.com. udp 42 false 4096\" NOERROR qr,rd,ra 188 0.020948545s", + "network.transport": "udp", + "service.type": "coredns", + "source.address": "::1", + "source.ip": "::1", + "source.port": 57413, + "tags": [ + "coredns" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/coredns/module.yml b/x-pack/filebeat/module/coredns/module.yml new file mode 100644 index 00000000000..e52f7f3de7f --- /dev/null +++ b/x-pack/filebeat/module/coredns/module.yml @@ -0,0 +1,3 @@ +dashboards: +- id: 53aa1f70-443e-11e9-8548-ab7fbe04f038 + file: Coredns-Overview-Dashboard.json \ No newline at end of file diff --git a/x-pack/filebeat/modules.d/coredns.yml.disabled b/x-pack/filebeat/modules.d/coredns.yml.disabled new file mode 100644 index 00000000000..d4a871455fd --- /dev/null +++ b/x-pack/filebeat/modules.d/coredns.yml.disabled @@ -0,0 +1,11 @@ +# Module: coredns +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-coredns.html + +- module: coredns + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: