From a0b96cf8f7149f51ed0016ae57427fd1152faf93 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Tue, 13 Dec 2016 15:07:53 +0100 Subject: [PATCH] Filebeat MySQL module (#3171) * Contains slowlog and errors filesets * Test files for two mysql versions (5.5 and 5.7) * Add support for built-in variables (e.g. `builtin.hostname`) * Contains a sample Kibana dashboard Part of #3159. --- filebeat/_meta/fields.common.yml | 5 + .../dashboard/Filebeat-MySQL-Dashboard.json | 13 + .../search/Filebeat-MySQL-Slow-log.json | 16 ++ .../search/Filebeat-MySQL-error-log.json | 17 ++ .../MySQL-Error-logs-levels.json | 11 + .../MySQL-Slow-logs-by-count.json | 11 + .../MySQL-Slow-queries-over-time.json | 11 + .../visualization/MySQL-error-logs.json | 11 + .../visualization/MySQL-slowest-queries.json | 11 + filebeat/docs/fields.asciidoc | 136 +++++++++++ filebeat/filebeat.py | 22 +- filebeat/filebeat.template-es2x.json | 83 +++++++ filebeat/filebeat.template.json | 73 ++++++ filebeat/module/mysql/_meta/fields.yml | 10 + filebeat/module/mysql/error/_meta/fields.yml | 21 ++ filebeat/module/mysql/error/config/error.yml | 6 + .../module/mysql/error/ingest/pipeline.json | 50 ++++ filebeat/module/mysql/error/manifest.yml | 11 + .../error/test/mysql-darwin-brew-5.7.10.log | 187 +++++++++++++++ .../mysql/error/test/mysql-ubuntu-5.5.53.log | 223 ++++++++++++++++++ .../module/mysql/slowlog/_meta/fields.yml | 42 ++++ .../module/mysql/slowlog/config/slowlog.yml | 10 + .../module/mysql/slowlog/ingest/pipeline.json | 39 +++ filebeat/module/mysql/slowlog/manifest.yml | 11 + .../slowlog/test/mysql-darwin-brew-5.7.10.log | 8 + .../slowlog/test/mysql-ubuntu-5.5.53.log | 66 ++++++ .../nginx/access/ingest/no_plugins.json | 6 + .../nginx/access/ingest/with_plugins.json | 6 + .../module/nginx/error/ingest/pipeline.json | 7 +- 29 files changed, 1119 insertions(+), 4 deletions(-) create mode 100644 filebeat/_meta/kibana/dashboard/Filebeat-MySQL-Dashboard.json create mode 100644 filebeat/_meta/kibana/search/Filebeat-MySQL-Slow-log.json create mode 100644 filebeat/_meta/kibana/search/Filebeat-MySQL-error-log.json create mode 100644 filebeat/_meta/kibana/visualization/MySQL-Error-logs-levels.json create mode 100644 filebeat/_meta/kibana/visualization/MySQL-Slow-logs-by-count.json create mode 100644 filebeat/_meta/kibana/visualization/MySQL-Slow-queries-over-time.json create mode 100644 filebeat/_meta/kibana/visualization/MySQL-error-logs.json create mode 100644 filebeat/_meta/kibana/visualization/MySQL-slowest-queries.json create mode 100644 filebeat/module/mysql/_meta/fields.yml create mode 100644 filebeat/module/mysql/error/_meta/fields.yml create mode 100644 filebeat/module/mysql/error/config/error.yml create mode 100644 filebeat/module/mysql/error/ingest/pipeline.json create mode 100644 filebeat/module/mysql/error/manifest.yml create mode 100644 filebeat/module/mysql/error/test/mysql-darwin-brew-5.7.10.log create mode 100644 filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53.log create mode 100644 filebeat/module/mysql/slowlog/_meta/fields.yml create mode 100644 filebeat/module/mysql/slowlog/config/slowlog.yml create mode 100644 filebeat/module/mysql/slowlog/ingest/pipeline.json create mode 100644 filebeat/module/mysql/slowlog/manifest.yml create mode 100644 filebeat/module/mysql/slowlog/test/mysql-darwin-brew-5.7.10.log create mode 100644 filebeat/module/mysql/slowlog/test/mysql-ubuntu-5.5.53.log diff --git a/filebeat/_meta/fields.common.yml b/filebeat/_meta/fields.common.yml index 79cc1cd5694..7c8d3331d1b 100644 --- a/filebeat/_meta/fields.common.yml +++ b/filebeat/_meta/fields.common.yml @@ -33,3 +33,8 @@ description: > The input type from which the event was generated. This field is set to the value specified for the `input_type` option in the prospector section of the Filebeat config file. + - name: error + description: > + Ingestion pipeline error message, added in case there are errors reported by + the Ingest Node in Elasticsearch. + diff --git a/filebeat/_meta/kibana/dashboard/Filebeat-MySQL-Dashboard.json b/filebeat/_meta/kibana/dashboard/Filebeat-MySQL-Dashboard.json new file mode 100644 index 00000000000..56e68d6c921 --- /dev/null +++ b/filebeat/_meta/kibana/dashboard/Filebeat-MySQL-Dashboard.json @@ -0,0 +1,13 @@ +{ + "hits": 0, + "timeRestore": false, + "description": "", + "title": "Filebeat MySQL Dashboard", + "uiStateJSON": "{\"P-1\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", + "panelsJSON": "[{\"col\":1,\"id\":\"MySQL-slowest-queries\",\"panelIndex\":1,\"row\":8,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-Slow-queries-over-time\",\"panelIndex\":2,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"MySQL-error-logs\",\"panelIndex\":3,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"columns\":[\"mysql.error.level\",\"mysql.error.message\"],\"id\":\"Filebeat-MySQL-error-log\",\"panelIndex\":4,\"row\":8,\"size_x\":6,\"size_y\":5,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":7,\"id\":\"MySQL-Error-logs-levels\",\"panelIndex\":5,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-Slow-logs-by-count\",\"panelIndex\":6,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"}]", + "optionsJSON": "{\"darkTheme\":false}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + } +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/search/Filebeat-MySQL-Slow-log.json b/filebeat/_meta/kibana/search/Filebeat-MySQL-Slow-log.json new file mode 100644 index 00000000000..ab26f37b6ba --- /dev/null +++ b/filebeat/_meta/kibana/search/Filebeat-MySQL-Slow-log.json @@ -0,0 +1,16 @@ +{ + "sort": [ + "@timestamp", + "desc" + ], + "hits": 0, + "description": "", + "title": "Filebeat MySQL Slow log", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"_exists_:mysql.slowlog\"}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + }, + "columns": [ + "_source" + ] +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/search/Filebeat-MySQL-error-log.json b/filebeat/_meta/kibana/search/Filebeat-MySQL-error-log.json new file mode 100644 index 00000000000..1e486476513 --- /dev/null +++ b/filebeat/_meta/kibana/search/Filebeat-MySQL-error-log.json @@ -0,0 +1,17 @@ +{ + "sort": [ + "@timestamp", + "desc" + ], + "hits": 0, + "description": "", + "title": "Filebeat MySQL error log", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"_exists_:mysql.error\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + }, + "columns": [ + "mysql.error.level", + "mysql.error.message" + ] +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/visualization/MySQL-Error-logs-levels.json b/filebeat/_meta/kibana/visualization/MySQL-Error-logs-levels.json new file mode 100644 index 00000000000..7c0c4c1a084 --- /dev/null +++ b/filebeat/_meta/kibana/visualization/MySQL-Error-logs-levels.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"MySQL Error logs levels\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"mysql.error.level\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "MySQL Error logs levels", + "uiStateJSON": "{\"vis\":{\"colors\":{\"Note\":\"#9AC48A\",\"Warning\":\"#F9934E\",\"ERROR\":\"#E24D42\"}}}", + "version": 1, + "savedSearchId": "Filebeat-MySQL-error-log", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/visualization/MySQL-Slow-logs-by-count.json b/filebeat/_meta/kibana/visualization/MySQL-Slow-logs-by-count.json new file mode 100644 index 00000000000..f9ad3ced5ec --- /dev/null +++ b/filebeat/_meta/kibana/visualization/MySQL-Slow-logs-by-count.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"MySQL Slow logs by count\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"mysql.slowlog.query\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "MySQL Slow logs by count", + "uiStateJSON": "{}", + "version": 1, + "savedSearchId": "Filebeat-MySQL-Slow-log", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/visualization/MySQL-Slow-queries-over-time.json b/filebeat/_meta/kibana/visualization/MySQL-Slow-queries-over-time.json new file mode 100644 index 00000000000..ce85abcf79f --- /dev/null +++ b/filebeat/_meta/kibana/visualization/MySQL-Slow-queries-over-time.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"MySQL Slow queries over time\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Slow queries\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "description": "", + "title": "MySQL Slow queries over time", + "uiStateJSON": "{\"vis\":{\"colors\":{\"Slow queries\":\"#EF843C\"}}}", + "version": 1, + "savedSearchId": "Filebeat-MySQL-Slow-log", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/visualization/MySQL-error-logs.json b/filebeat/_meta/kibana/visualization/MySQL-error-logs.json new file mode 100644 index 00000000000..99ff00e2a1b --- /dev/null +++ b/filebeat/_meta/kibana/visualization/MySQL-error-logs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"MySQL error logs\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Error logs\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "description": "", + "title": "MySQL error logs", + "uiStateJSON": "{\"vis\":{\"colors\":{\"Count\":\"#447EBC\",\"Error logs\":\"#1F78C1\"}}}", + "version": 1, + "savedSearchId": "Filebeat-MySQL-error-log", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } +} \ No newline at end of file diff --git a/filebeat/_meta/kibana/visualization/MySQL-slowest-queries.json b/filebeat/_meta/kibana/visualization/MySQL-slowest-queries.json new file mode 100644 index 00000000000..82ce2acaedb --- /dev/null +++ b/filebeat/_meta/kibana/visualization/MySQL-slowest-queries.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"MySQL slowest queries\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"mysql.slowlog.query_time.sec\",\"customLabel\":\"Query time\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"mysql.slowlog.query\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Query\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"mysql.slowlog.user\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"User\"}}],\"listeners\":{}}", + "description": "", + "title": "MySQL slowest queries", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "savedSearchId": "Filebeat-MySQL-Slow-log", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } +} \ No newline at end of file diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index f807f668dfd..f95ca899918 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -15,6 +15,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> -- @@ -175,6 +176,141 @@ required: True The input type from which the event was generated. This field is set to the value specified for the `input_type` option in the prospector section of the Filebeat config file. +[float] +=== error + +Ingestion pipeline error message, added in case there are errors reported by the Ingest Node in Elasticsearch. + + +[[exported-fields-mysql]] +== MySQL Fields + +Module for parsing the MySQL log files. + + + +[float] +== mysql Fields + +Fields from the MySQL log files. + + + +[float] +== error Fields + +Contains fields from the MySQL error logs. + + + +[float] +=== mysql.error.timestamp + +The timestamp from the log line. + + +[float] +=== mysql.error.thread_id + +type: long + +As of MySQL 5.7.2, this is the thread id. For MySQL versions prior to 5.7.2, this field contains the process id. + + +[float] +=== mysql.error.level + +example: Warning + +The log level. + +[float] +=== mysql.error.message + +type: text + +The logged message. + + +[float] +== slowlog Fields + +Contains fields from the MySQL slow logs. + + + +[float] +=== mysql.slowlog.user + +The MySQL user that created the query. + + +[float] +=== mysql.slowlog.host + +The host from where the user that created the query logged in. + + +[float] +=== mysql.slowlog.ip + +The IP address from where the user that created the query logged in. + + +[float] +=== mysql.slowlog.query_time.sec + +type: float + +The total time the query took, in seconds, as a floating point number. + + +[float] +=== mysql.slowlog.lock_time.sec + +type: float + +The amount of time the query waited for the lock to be available. The value is in seconds, as a floating point number. + + +[float] +=== mysql.slowlog.rows_sent + +type: long + +The number of rows returned by the query. + + +[float] +=== mysql.slowlog.rows_examined + +type: long + +The number of rows scanned by the query. + + +[float] +=== mysql.slowlog.timestamp + +type: long + +The unix timestamp taken from the `SET timestamp` query. + + +[float] +=== mysql.slowlog.query + +The slow query. + + +[float] +=== mysql.slowlog.id + +type: long + +The connection ID for the query. + + [[exported-fields-nginx]] == Nginx Fields diff --git a/filebeat/filebeat.py b/filebeat/filebeat.py index 4bd7aef9c7c..2ba6b2b395e 100755 --- a/filebeat/filebeat.py +++ b/filebeat/filebeat.py @@ -6,6 +6,7 @@ import requests import tempfile import subprocess +import socket from jinja2 import Template @@ -37,7 +38,7 @@ def main(): def load_dashboards(args): cmd = ["../libbeat/dashboards/import_dashboards", - "-dir", "etc/kibana", + "-dir", "_meta/kibana", "-es", args.es] subprocess.Popen(cmd).wait() @@ -77,7 +78,9 @@ def load_fileset(args, module, fileset, path): def evaluate_vars(args, var_in): - var = {} + var = { + "builtin": get_builtin_vars() + } for name, vals in var_in.items(): var[name] = vals["default"] @@ -86,6 +89,8 @@ def evaluate_vars(args, var_in): elif sys.platform == "windows" and "os.windows" in vals: var[name] = vals["os.windows"] + var[name] = Template(var[name]).render(var) + # overrides if args.E is not None: for pair in args.E: @@ -95,6 +100,16 @@ def evaluate_vars(args, var_in): return var +def get_builtin_vars(): + host = socket.gethostname() + hostname, _, domain = host.partition(".") + # separate the domain + return { + "hostname": hostname, + "domain": domain + } + + def load_pipeline(var, pipeline): path = os.path.join(var["beat"]["path"], Template(pipeline).render(var)) print("Loading ingest pipeline: {}".format(path)) @@ -110,7 +125,8 @@ def load_pipeline(var, pipeline): var["beat"]["pipeline_id"]), data=contents) if r.status_code >= 300: - print("Error posting template: {}".format(r.text)) + print("Error posting pipeline: {}".format(r.text)) + sys.exit(1) def run_filebeat(args, prospectors): diff --git a/filebeat/filebeat.template-es2x.json b/filebeat/filebeat.template-es2x.json index 294cf6d5388..99e5a5e5efe 100644 --- a/filebeat/filebeat.template-es2x.json +++ b/filebeat/filebeat.template-es2x.json @@ -44,6 +44,11 @@ } } }, + "error": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, "input_type": { "ignore_above": 1024, "index": "not_analyzed", @@ -94,6 +99,84 @@ } } }, + "mysql": { + "properties": { + "error": { + "properties": { + "level": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "message": { + "index": "analyzed", + "norms": { + "enabled": false + }, + "type": "string" + }, + "thread_id": { + "type": "long" + }, + "timestamp": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + }, + "slowlog": { + "properties": { + "host": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "id": { + "type": "long" + }, + "ip": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "lock_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "query": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "query_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "rows_examined": { + "type": "long" + }, + "rows_sent": { + "type": "long" + }, + "timestamp": { + "type": "long" + }, + "user": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + } + } + }, "nginx": { "properties": { "access": { diff --git a/filebeat/filebeat.template.json b/filebeat/filebeat.template.json index d1e8858e8a3..f1b18e79fda 100644 --- a/filebeat/filebeat.template.json +++ b/filebeat/filebeat.template.json @@ -38,6 +38,10 @@ } } }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, "input_type": { "ignore_above": 1024, "type": "keyword" @@ -78,6 +82,75 @@ } } }, + "mysql": { + "properties": { + "error": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "thread_id": { + "type": "long" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + }, + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "lock_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "query_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "rows_examined": { + "type": "long" + }, + "rows_sent": { + "type": "long" + }, + "timestamp": { + "type": "long" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, "nginx": { "properties": { "access": { diff --git a/filebeat/module/mysql/_meta/fields.yml b/filebeat/module/mysql/_meta/fields.yml new file mode 100644 index 00000000000..4cafbcd8745 --- /dev/null +++ b/filebeat/module/mysql/_meta/fields.yml @@ -0,0 +1,10 @@ +- key: mysql + title: "MySQL" + description: > + Module for parsing the MySQL log files. + fields: + - name: mysql + type: group + description: > + Fields from the MySQL log files. + fields: diff --git a/filebeat/module/mysql/error/_meta/fields.yml b/filebeat/module/mysql/error/_meta/fields.yml new file mode 100644 index 00000000000..c878689f968 --- /dev/null +++ b/filebeat/module/mysql/error/_meta/fields.yml @@ -0,0 +1,21 @@ +- name: error + type: group + description: > + Contains fields from the MySQL error logs. + fields: + - name: timestamp + description: > + The timestamp from the log line. + - name: thread_id + type: long + description: > + As of MySQL 5.7.2, this is the thread id. For MySQL versions prior to 5.7.2, this + field contains the process id. + - name: level + example: "Warning" + description: + The log level. + - name: message + type: text + description: > + The logged message. diff --git a/filebeat/module/mysql/error/config/error.yml b/filebeat/module/mysql/error/config/error.yml new file mode 100644 index 00000000000..64aba84999f --- /dev/null +++ b/filebeat/module/mysql/error/config/error.yml @@ -0,0 +1,6 @@ +- input_type: log + paths: + - {{path}} + fields: + source_type: mysql-error + pipeline_id: {{beat.pipeline_id}} diff --git a/filebeat/module/mysql/error/ingest/pipeline.json b/filebeat/module/mysql/error/ingest/pipeline.json new file mode 100644 index 00000000000..f8a7faf73ad --- /dev/null +++ b/filebeat/module/mysql/error/ingest/pipeline.json @@ -0,0 +1,50 @@ +{ + "description": "Pipeline for parsing MySQL error logs. It currently uses message1 and message2 to workaround bug https://github.com/elastic/elasticsearch/issues/22117", + "processors": [{ + "grok": { + "field": "message", + "patterns": [ + "%{LOCALDATETIME:mysql.error.timestamp} (\\[%{DATA:mysql.error.level}\\] )?%{GREEDYDATA:mysql.error.message}", + "%{DATA:mysql.error.timestamp} %{NUMBER:mysql.error.id} \\[%{DATA:mysql.error.level}\\] %{GREEDYDATA:mysql.error.message1}", + "%{GREEDYDATA:mysql.error.message2}" + ], + "ignore_missing": true, + "pattern_definitions": { + "LOCALDATETIME": "[0-9]+ %{TIME}", + "GREEDYDATA1": ".*" + } + } + }, { + "remove": { + "field": "message" + } + }, { + "rename": { + "field": "mysql.error.message1", + "target_field": "mysql.error.message", + "ignore_failure": true + } + }, { + "rename": { + "field": "mysql.error.message2", + "target_field": "mysql.error.message", + "ignore_failure": true + } + }, { + "date": { + "field": "mysql.error.timestamp", + "target_field": "@timestamp", + "formats": [ + "ISO8601", + "YYMMdd H:m:s" + ], + "ignore_failure": true + } + }], + "on_failure" : [{ + "set" : { + "field" : "error", + "value" : "{{ _ingest.on_failure_message }}" + } + }] +} diff --git a/filebeat/module/mysql/error/manifest.yml b/filebeat/module/mysql/error/manifest.yml new file mode 100644 index 00000000000..929eede06c0 --- /dev/null +++ b/filebeat/module/mysql/error/manifest.yml @@ -0,0 +1,11 @@ +module_version: 1.0 + +vars: + path: + default: /var/log/mysql/error.log + os.darwin: /usr/local/var/mysql/{{builtin.hostname}}.{{builtin.domain}}.err + os.windows: "c:/programdata/MySQL/MySQL Server*/error.log" + +ingest_pipeline: ingest/pipeline.json +prospectors: + - config/error.yml diff --git a/filebeat/module/mysql/error/test/mysql-darwin-brew-5.7.10.log b/filebeat/module/mysql/error/test/mysql-darwin-brew-5.7.10.log new file mode 100644 index 00000000000..5d9a6d01f58 --- /dev/null +++ b/filebeat/module/mysql/error/test/mysql-darwin-brew-5.7.10.log @@ -0,0 +1,187 @@ +161209 13:08:33 mysqld_safe Starting mysqld daemon with databases from /usr/local/var/mysql +2016-12-09T12:08:33.335060Z 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details). +2016-12-09T12:08:33.335892Z 0 [Warning] Insecure configuration for --secure-file-priv: Current value does not restrict location of generated files. Consider setting it to a valid, non-empty path. +2016-12-09T12:08:33.336610Z 0 [Note] /usr/local/Cellar/mysql/5.7.10/bin/mysqld (mysqld 5.7.10) starting as process 61571 ... +2016-12-09T12:08:33.345527Z 0 [Warning] Setting lower_case_table_names=2 because file system for /usr/local/var/mysql/ is case insensitive +2016-12-09T12:08:33.351596Z 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins +2016-12-09T12:08:33.351632Z 0 [Note] InnoDB: Uses event mutexes +2016-12-09T12:08:33.351638Z 0 [Note] InnoDB: GCC builtin __atomic_thread_fence() is used for memory barrier +2016-12-09T12:08:33.351641Z 0 [Note] InnoDB: Compressed tables use zlib 1.2.3 +2016-12-09T12:08:33.352714Z 0 [Note] InnoDB: Number of pools: 1 +2016-12-09T12:08:33.354153Z 0 [Note] InnoDB: Using CPU crc32 instructions +2016-12-09T12:08:33.366818Z 0 [Note] InnoDB: Initializing buffer pool, total size = 128M, instances = 1, chunk size = 128M +2016-12-09T12:08:33.379566Z 0 [Note] InnoDB: Completed initialization of buffer pool +2016-12-09T12:08:33.401031Z 0 [Note] InnoDB: Highest supported file format is Barracuda. +2016-12-09T12:08:33.402267Z 0 [Note] InnoDB: Log scan progressed past the checkpoint lsn 2498863 +2016-12-09T12:08:33.402289Z 0 [Note] InnoDB: Doing recovery: scanned up to log sequence number 2498872 +2016-12-09T12:08:33.402433Z 0 [Note] InnoDB: Doing recovery: scanned up to log sequence number 2498872 +2016-12-09T12:08:33.402440Z 0 [Note] InnoDB: Database was not shutdown normally! +2016-12-09T12:08:33.402443Z 0 [Note] InnoDB: Starting crash recovery. +2016-12-09T12:08:33.549180Z 0 [Note] InnoDB: Removed temporary tablespace data file: "ibtmp1" +2016-12-09T12:08:33.549206Z 0 [Note] InnoDB: Creating shared tablespace for temporary tables +2016-12-09T12:08:33.549317Z 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ... +2016-12-09T12:08:33.585175Z 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB. +2016-12-09T12:08:33.588126Z 0 [Note] InnoDB: 96 redo rollback segment(s) found. 96 redo rollback segment(s) are active. +2016-12-09T12:08:33.588151Z 0 [Note] InnoDB: 32 non-redo rollback segment(s) are active. +2016-12-09T12:08:33.588453Z 0 [Note] InnoDB: Waiting for purge to start +2016-12-09T12:08:33.641851Z 0 [Note] InnoDB: 5.7.10 started; log sequence number 2498872 +2016-12-09T12:08:33.642289Z 0 [Note] InnoDB: Loading buffer pool(s) from /usr/local/var/mysql/ib_buffer_pool +2016-12-09T12:08:33.642503Z 0 [Note] InnoDB: not started +2016-12-09T12:08:33.643066Z 0 [Note] Plugin 'FEDERATED' is disabled. +2016-12-09T12:08:33.652949Z 0 [Note] InnoDB: Buffer pool(s) load completed at 161209 13:08:33 +2016-12-09T12:08:33.662950Z 0 [Note] Found ca.pem, server-cert.pem and server-key.pem in data directory. Trying to enable SSL support using them. +2016-12-09T12:08:33.662974Z 0 [Note] Skipping generation of SSL certificates as certificate files are present in data directory. +2016-12-09T12:08:33.665405Z 0 [Warning] CA certificate ca.pem is self signed. +2016-12-09T12:08:33.665842Z 0 [Note] Skipping generation of RSA key pair as key files are present in data directory. +2016-12-09T12:08:33.698843Z 0 [Note] Server hostname (bind-address): '*'; port: 3306 +2016-12-09T12:08:33.699354Z 0 [Note] IPv6 is available. +2016-12-09T12:08:33.699553Z 0 [Note] - '::' resolves to '::'; +2016-12-09T12:08:33.699665Z 0 [Note] Server socket created on IP: '::'. +2016-12-09T12:08:33.784625Z 0 [Note] Event Scheduler: Loaded 0 events +2016-12-09T12:08:33.784722Z 0 [Note] /usr/local/Cellar/mysql/5.7.10/bin/mysqld: ready for connections. +Version: '5.7.10' socket: '/tmp/mysql.sock' port: 3306 Homebrew +2016-12-09T22:21:02.443689Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 772568ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-09T22:36:49.017929Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 898642ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-09T23:37:34.021038Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3596603ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T00:17:54.198501Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 2371678ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T01:18:38.017222Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3597590ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T01:39:00.017683Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 1173583ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T02:39:45.021071Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3597610ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T02:49:08.015573Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515469ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T03:24:15.016664Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 2059611ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T04:25:00.016866Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3597614ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T04:34:24.021797Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515589ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T04:39:18.022366Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 246613ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T05:40:03.016821Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3598614ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T06:40:48.025959Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3595608ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T06:45:55.018094Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 258594ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T07:46:40.016090Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3598632ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T07:56:04.016254Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515603ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T08:56:49.390467Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3597607ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T09:06:11.019025Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515633ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T10:06:56.015782Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3597617ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T10:16:18.022349Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 514638ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T11:17:02.165133Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3595614ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T11:30:44.018136Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 773594ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T12:03:24.017458Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 1912617ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T12:06:40.015089Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 150375ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T12:24:37.025219Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 1030636ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T13:25:22.017729Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3596603ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T13:39:05.016348Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 774598ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T14:39:50.178488Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3597787ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T14:49:14.023629Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515462ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T15:49:59.022762Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3597628ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T15:59:23.014556Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515609ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T17:00:08.019274Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3598607ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T17:09:30.026900Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515633ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T17:48:20.017991Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 2282610ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T18:00:05.183218Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515227ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T18:54:13.016955Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3200608ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T20:13:03.016738Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3089523ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T20:50:11.201413Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 2180623ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T20:53:54.016961Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 176629ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T21:03:18.023529Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 516622ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T22:04:03.021479Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3598602ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T22:13:57.015306Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 545611ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T22:49:59.020252Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 2114631ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-10T23:12:12.023076Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 1287614ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T00:12:57.015297Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3595581ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T00:26:41.053666Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 773622ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T00:47:44.015939Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 1215572ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T00:49:50.017378Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 79642ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T01:20:40.031015Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 1803651ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T02:21:24.021184Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3595607ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T02:26:30.015089Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 257596ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T03:18:55.018844Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3097591ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T04:15:14.022467Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3331614ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T04:20:52.016318Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 289611ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T04:25:56.035073Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 257653ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T05:26:41.020131Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3598198ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T05:36:05.024444Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515624ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T06:36:50.017205Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3598619ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T06:46:14.016702Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515611ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T07:22:43.020901Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 2141603ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T07:26:22.018814Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 172601ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T07:35:45.031855Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 516617ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T08:36:30.015495Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3597620ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T08:45:54.017653Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515622ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T09:46:39.018813Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3598618ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T09:56:03.014615Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515592ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T10:56:48.042131Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3598651ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T11:06:12.025718Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515588ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T12:06:56.018725Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3597614ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T12:16:19.014169Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 515625ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T13:17:04.016183Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3597594ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T13:26:28.023088Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 514629ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T14:27:13.164509Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3597613ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T14:36:38.015297Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 514596ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T15:01:44.026173Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 1460642ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T15:15:28.015652Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 773648ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T16:16:13.027289Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3597657ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T16:30:51.021667Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 813477ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T17:18:43.027126Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 2824646ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T21:47:54.189105Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 592456ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T22:10:17.210550Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 1309781ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T23:11:02.043893Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3596681ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-11T23:24:54.163597Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 782633ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T00:25:39.024180Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3597668ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T00:35:03.029112Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 514635ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T01:07:31.173633Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 1900621ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T01:10:46.030187Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 147678ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T01:28:43.027567Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 1029630ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T02:29:28.163850Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3595664ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T03:17:29.198796Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 2831638ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T03:27:41.042627Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 564697ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T03:29:27.172837Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 58662ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T04:30:12.026414Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3595651ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T04:35:19.040164Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 257645ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T05:24:10.022697Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 2883673ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T05:43:00.028615Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 1060590ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T06:43:49.185223Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3596652ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T06:48:27.033604Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 228658ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T06:55:03.184509Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 257407ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +2016-12-12T07:55:49.022710Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 3598669ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +161212 12:40:38 mysqld_safe Starting mysqld daemon with databases from /usr/local/var/mysql +2016-12-12T11:40:39.000639Z 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details). +2016-12-12T11:40:39.001684Z 0 [Warning] 'NO_ZERO_DATE', 'NO_ZERO_IN_DATE' and 'ERROR_FOR_DIVISION_BY_ZERO' sql modes should be used with strict mode. They will be merged with strict mode in a future release. +2016-12-12T11:40:39.001726Z 0 [Warning] 'NO_AUTO_CREATE_USER' sql mode was not set. +2016-12-12T11:40:39.001942Z 0 [Warning] Insecure configuration for --secure-file-priv: Current value does not restrict location of generated files. Consider setting it to a valid, non-empty path. +2016-12-12T11:40:39.003193Z 0 [Note] /usr/local/Cellar/mysql/5.7.10/bin/mysqld (mysqld 5.7.10) starting as process 97910 ... +2016-12-12T11:40:39.012775Z 0 [Warning] Setting lower_case_table_names=2 because file system for /usr/local/var/mysql/ is case insensitive +2016-12-12T11:40:39.020465Z 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins +2016-12-12T11:40:39.020530Z 0 [Note] InnoDB: Uses event mutexes +2016-12-12T11:40:39.020540Z 0 [Note] InnoDB: GCC builtin __atomic_thread_fence() is used for memory barrier +2016-12-12T11:40:39.020545Z 0 [Note] InnoDB: Compressed tables use zlib 1.2.3 +2016-12-12T11:40:39.021852Z 0 [Note] InnoDB: Number of pools: 1 +2016-12-12T11:40:39.023939Z 0 [Note] InnoDB: Using CPU crc32 instructions +2016-12-12T11:40:39.037980Z 0 [Note] InnoDB: Initializing buffer pool, total size = 128M, instances = 1, chunk size = 128M +2016-12-12T11:40:39.055856Z 0 [Note] InnoDB: Completed initialization of buffer pool +2016-12-12T11:40:39.079839Z 0 [Note] InnoDB: Highest supported file format is Barracuda. +2016-12-12T11:40:39.082943Z 0 [Note] InnoDB: Log scan progressed past the checkpoint lsn 2498891 +2016-12-12T11:40:39.082997Z 0 [Note] InnoDB: Doing recovery: scanned up to log sequence number 2498900 +2016-12-12T11:40:39.083372Z 0 [Note] InnoDB: Doing recovery: scanned up to log sequence number 2498900 +2016-12-12T11:40:39.083404Z 0 [Note] InnoDB: Database was not shutdown normally! +2016-12-12T11:40:39.083412Z 0 [Note] InnoDB: Starting crash recovery. +2016-12-12T11:40:39.243369Z 0 [Note] InnoDB: Removed temporary tablespace data file: "ibtmp1" +2016-12-12T11:40:39.243411Z 0 [Note] InnoDB: Creating shared tablespace for temporary tables +2016-12-12T11:40:39.243762Z 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ... +2016-12-12T11:40:39.262976Z 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB. +2016-12-12T11:40:39.264392Z 0 [Note] InnoDB: 96 redo rollback segment(s) found. 96 redo rollback segment(s) are active. +2016-12-12T11:40:39.264418Z 0 [Note] InnoDB: 32 non-redo rollback segment(s) are active. +2016-12-12T11:40:39.264631Z 0 [Note] InnoDB: Waiting for purge to start +2016-12-12T11:40:39.318411Z 0 [Note] InnoDB: 5.7.10 started; log sequence number 2498900 +2016-12-12T11:40:39.319011Z 0 [Note] InnoDB: Loading buffer pool(s) from /usr/local/var/mysql/ib_buffer_pool +2016-12-12T11:40:39.319443Z 0 [Note] InnoDB: not started +2016-12-12T11:40:39.319804Z 0 [Note] Plugin 'FEDERATED' is disabled. +2016-12-12T11:40:39.330889Z 0 [Note] InnoDB: Buffer pool(s) load completed at 161212 12:40:39 +2016-12-12T11:40:39.343948Z 0 [Note] Found ca.pem, server-cert.pem and server-key.pem in data directory. Trying to enable SSL support using them. +2016-12-12T11:40:39.343980Z 0 [Note] Skipping generation of SSL certificates as certificate files are present in data directory. +2016-12-12T11:40:39.346921Z 0 [Warning] CA certificate ca.pem is self signed. +2016-12-12T11:40:39.347342Z 0 [Note] Skipping generation of RSA key pair as key files are present in data directory. +2016-12-12T11:40:39.362897Z 0 [Note] Server hostname (bind-address): '*'; port: 3306 +2016-12-12T11:40:39.363163Z 0 [Note] IPv6 is available. +2016-12-12T11:40:39.363202Z 0 [Note] - '::' resolves to '::'; +2016-12-12T11:40:39.363224Z 0 [Note] Server socket created on IP: '::'. +2016-12-12T11:40:39.540292Z 0 [Note] Event Scheduler: Loaded 0 events +2016-12-12T11:40:39.540498Z 0 [Note] /usr/local/Cellar/mysql/5.7.10/bin/mysqld: ready for connections. +Version: '5.7.10' socket: '/tmp/mysql.sock' port: 3306 Homebrew diff --git a/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53.log b/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53.log new file mode 100644 index 00000000000..1e9ed334a1f --- /dev/null +++ b/filebeat/module/mysql/error/test/mysql-ubuntu-5.5.53.log @@ -0,0 +1,223 @@ +161209 14:18:50 [Warning] Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead. +161209 14:18:50 [Note] Plugin 'FEDERATED' is disabled. +161209 14:18:50 InnoDB: The InnoDB memory heap is disabled +161209 14:18:50 InnoDB: Mutexes and rw_locks use GCC atomic builtins +161209 14:18:50 InnoDB: Compressed tables use zlib 1.2.3.4 +161209 14:18:50 InnoDB: Initializing buffer pool, size = 128.0M +161209 14:18:50 InnoDB: Completed initialization of buffer pool +InnoDB: The first specified data file ./ibdata1 did not exist: +InnoDB: a new database to be created! +161209 14:18:50 InnoDB: Setting file ./ibdata1 size to 10 MB +InnoDB: Database physically writes the file full: wait... +161209 14:18:50 InnoDB: Log file ./ib_logfile0 did not exist: new to be created +InnoDB: Setting log file ./ib_logfile0 size to 5 MB +InnoDB: Database physically writes the file full: wait... +161209 14:18:50 InnoDB: Log file ./ib_logfile1 did not exist: new to be created +InnoDB: Setting log file ./ib_logfile1 size to 5 MB +InnoDB: Database physically writes the file full: wait... +InnoDB: Doublewrite buffer not found: creating new +InnoDB: Doublewrite buffer created +InnoDB: 127 rollback segment(s) active. +InnoDB: Creating foreign key constraint system tables +InnoDB: Foreign key constraint system tables created +161209 14:18:50 InnoDB: Waiting for the background threads to start +161209 14:18:51 InnoDB: 5.5.53 started; log sequence number 0 +161209 14:18:51 InnoDB: Starting shutdown... +161209 14:18:52 InnoDB: Shutdown completed; log sequence number 1595675 +161209 14:18:52 [Warning] Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead. +161209 14:18:52 [Note] Plugin 'FEDERATED' is disabled. +161209 14:18:52 InnoDB: The InnoDB memory heap is disabled +161209 14:18:52 InnoDB: Mutexes and rw_locks use GCC atomic builtins +161209 14:18:52 InnoDB: Compressed tables use zlib 1.2.3.4 +161209 14:18:52 InnoDB: Initializing buffer pool, size = 128.0M +161209 14:18:52 InnoDB: Completed initialization of buffer pool +161209 14:18:52 InnoDB: highest supported file format is Barracuda. +161209 14:18:52 InnoDB: Waiting for the background threads to start +161209 14:18:53 InnoDB: 5.5.53 started; log sequence number 1595675 +ERROR: 1064 You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'ALTER TABLE user ADD column Show_view_priv enum('N','Y') CHARACTER SET utf8 NOT ' at line 1 +161209 14:18:53 [ERROR] Aborting + +161209 14:18:53 InnoDB: Starting shutdown... +161209 14:18:53 InnoDB: Shutdown completed; log sequence number 1595675 +161209 14:18:53 [Note] /usr/sbin/mysqld: Shutdown complete + +161209 14:18:53 [Warning] Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead. +161209 14:18:53 [Note] Plugin 'FEDERATED' is disabled. +161209 14:18:53 InnoDB: The InnoDB memory heap is disabled +161209 14:18:53 InnoDB: Mutexes and rw_locks use GCC atomic builtins +161209 14:18:53 InnoDB: Compressed tables use zlib 1.2.3.4 +161209 14:18:53 InnoDB: Initializing buffer pool, size = 128.0M +161209 14:18:53 InnoDB: Completed initialization of buffer pool +161209 14:18:53 InnoDB: highest supported file format is Barracuda. +161209 14:18:53 InnoDB: Waiting for the background threads to start +161209 14:18:54 InnoDB: 5.5.53 started; log sequence number 1595675 +161209 14:18:54 InnoDB: Starting shutdown... +161209 14:18:56 [ERROR] Aborting + +161209 14:18:56 InnoDB: Starting shutdown... +161209 14:18:56 InnoDB: Shutdown completed; log sequence number 1595675 +161209 14:18:56 [Note] /usr/sbin/mysqld: Shutdown complete + +161209 14:18:56 [Warning] Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead. +161209 14:18:56 [Note] Plugin 'FEDERATED' is disabled. +161209 14:18:56 InnoDB: The InnoDB memory heap is disabled +161209 14:18:56 InnoDB: Mutexes and rw_locks use GCC atomic builtins +161209 14:18:56 InnoDB: Compressed tables use zlib 1.2.3.4 +161209 14:18:56 InnoDB: Initializing buffer pool, size = 128.0M +161209 14:18:56 InnoDB: Completed initialization of buffer pool +161209 14:18:57 InnoDB: highest supported file format is Barracuda. +161209 14:18:57 InnoDB: Waiting for the background threads to start +161209 14:18:58 InnoDB: 5.5.53 started; log sequence number 1595675 +161209 14:18:58 [Note] Server hostname (bind-address): '127.0.0.1'; port: 3306 +161209 14:18:58 [Note] - '127.0.0.1' resolves to '127.0.0.1'; +161209 14:18:58 [Note] Server socket created on IP: '127.0.0.1'. +161209 14:18:58 [Note] Event Scheduler: Loaded 0 events +161209 14:18:58 [Note] /usr/sbin/mysqld: ready for connections. +Version: '5.5.53-0ubuntu0.12.04.1' socket: '/var/run/mysqld/mysqld.sock' port: 3306 (Ubuntu) +161209 14:37:57 [Note] /usr/sbin/mysqld: Normal shutdown + +161209 14:37:57 [Note] Event Scheduler: Purging the queue. 0 events +161209 14:37:57 InnoDB: Starting shutdown... +161209 14:37:57 InnoDB: Shutdown completed; log sequence number 1595685 +161209 14:37:57 [Note] /usr/sbin/mysqld: Shutdown complete + +161209 14:37:57 [Warning] Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead. +161209 14:37:57 [Note] Plugin 'FEDERATED' is disabled. +161209 14:37:57 InnoDB: The InnoDB memory heap is disabled +161209 14:37:57 InnoDB: Mutexes and rw_locks use GCC atomic builtins +161209 14:37:57 InnoDB: Compressed tables use zlib 1.2.3.4 +161209 14:37:57 InnoDB: Initializing buffer pool, size = 128.0M +161209 14:37:57 InnoDB: Completed initialization of buffer pool +161209 14:37:57 InnoDB: highest supported file format is Barracuda. +161209 14:37:57 InnoDB: Waiting for the background threads to start +161209 14:37:58 InnoDB: 5.5.53 started; log sequence number 1595685 +161209 14:37:58 [Note] Server hostname (bind-address): '127.0.0.1'; port: 3306 +161209 14:37:58 [Note] - '127.0.0.1' resolves to '127.0.0.1'; +161209 14:37:58 [Note] Server socket created on IP: '127.0.0.1'. +161209 14:37:58 [Note] Event Scheduler: Loaded 0 events +161209 14:37:58 [Note] /usr/sbin/mysqld: ready for connections. +Version: '5.5.53-0ubuntu0.12.04.1-log' socket: '/var/run/mysqld/mysqld.sock' port: 3306 (Ubuntu) +vagrant@precise32:~$ cat /var/log/mysql.log | grep phisically +vagrant@precise32:~$ cat /var/log/mysql.log | grep physi +vagrant@precise32:~$ cat /var/log/mysql.log | physically +physically: command not found +vagrant@precise32:~$ cat /var/log/mysql.log | grep physically +vagrant@precise32:~$ less /var/log/mysql. +mysql.err mysql.log +vagrant@precise32:~$ less /var/log/mysql.err +vagrant@precise32:~$ less /var/log/mysql.log +vagrant@precise32:~$ less /var/log/mysql/ +error.log mysql-slow.log +vagrant@precise32:~$ less /var/log/mysql/error.log +vagrant@precise32:~$ cat /var/log/mysql/error.log +161209 14:18:50 [Warning] Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead. +161209 14:18:50 [Note] Plugin 'FEDERATED' is disabled. +161209 14:18:50 InnoDB: The InnoDB memory heap is disabled +161209 14:18:50 InnoDB: Mutexes and rw_locks use GCC atomic builtins +161209 14:18:50 InnoDB: Compressed tables use zlib 1.2.3.4 +161209 14:18:50 InnoDB: Initializing buffer pool, size = 128.0M +161209 14:18:50 InnoDB: Completed initialization of buffer pool +InnoDB: The first specified data file ./ibdata1 did not exist: +InnoDB: a new database to be created! +161209 14:18:50 InnoDB: Setting file ./ibdata1 size to 10 MB +InnoDB: Database physically writes the file full: wait... +161209 14:18:50 InnoDB: Log file ./ib_logfile0 did not exist: new to be created +InnoDB: Setting log file ./ib_logfile0 size to 5 MB +InnoDB: Database physically writes the file full: wait... +161209 14:18:50 InnoDB: Log file ./ib_logfile1 did not exist: new to be created +InnoDB: Setting log file ./ib_logfile1 size to 5 MB +InnoDB: Database physically writes the file full: wait... +InnoDB: Doublewrite buffer not found: creating new +InnoDB: Doublewrite buffer created +InnoDB: 127 rollback segment(s) active. +InnoDB: Creating foreign key constraint system tables +InnoDB: Foreign key constraint system tables created +161209 14:18:50 InnoDB: Waiting for the background threads to start +161209 14:18:51 InnoDB: 5.5.53 started; log sequence number 0 +161209 14:18:51 InnoDB: Starting shutdown... +161209 14:18:52 InnoDB: Shutdown completed; log sequence number 1595675 +161209 14:18:52 [Warning] Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead. +161209 14:18:52 [Note] Plugin 'FEDERATED' is disabled. +161209 14:18:52 InnoDB: The InnoDB memory heap is disabled +161209 14:18:52 InnoDB: Mutexes and rw_locks use GCC atomic builtins +161209 14:18:52 InnoDB: Compressed tables use zlib 1.2.3.4 +161209 14:18:52 InnoDB: Initializing buffer pool, size = 128.0M +161209 14:18:52 InnoDB: Completed initialization of buffer pool +161209 14:18:52 InnoDB: highest supported file format is Barracuda. +161209 14:18:52 InnoDB: Waiting for the background threads to start +161209 14:18:53 InnoDB: 5.5.53 started; log sequence number 1595675 +ERROR: 1064 You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'ALTER TABLE user ADD column Show_view_priv enum('N','Y') CHARACTER SET utf8 NOT ' at line 1 +161209 14:18:53 [ERROR] Aborting + +161209 14:18:53 InnoDB: Starting shutdown... +161209 14:18:53 InnoDB: Shutdown completed; log sequence number 1595675 +161209 14:18:53 [Note] /usr/sbin/mysqld: Shutdown complete + +161209 14:18:53 [Warning] Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead. +161209 14:18:53 [Note] Plugin 'FEDERATED' is disabled. +161209 14:18:53 InnoDB: The InnoDB memory heap is disabled +161209 14:18:53 InnoDB: Mutexes and rw_locks use GCC atomic builtins +161209 14:18:53 InnoDB: Compressed tables use zlib 1.2.3.4 +161209 14:18:53 InnoDB: Initializing buffer pool, size = 128.0M +161209 14:18:53 InnoDB: Completed initialization of buffer pool +161209 14:18:53 InnoDB: highest supported file format is Barracuda. +161209 14:18:53 InnoDB: Waiting for the background threads to start +161209 14:18:54 InnoDB: 5.5.53 started; log sequence number 1595675 +161209 14:18:54 InnoDB: Starting shutdown... +161209 14:18:55 InnoDB: Shutdown completed; log sequence number 1595675 +161209 14:18:55 [Warning] Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead. +161209 14:18:55 [Note] Plugin 'FEDERATED' is disabled. +161209 14:18:55 InnoDB: The InnoDB memory heap is disabled +161209 14:18:55 InnoDB: Mutexes and rw_locks use GCC atomic builtins +161209 14:18:55 InnoDB: Compressed tables use zlib 1.2.3.4 +161209 14:18:55 InnoDB: Initializing buffer pool, size = 128.0M +161209 14:18:55 InnoDB: Completed initialization of buffer pool +161209 14:18:55 InnoDB: highest supported file format is Barracuda. +161209 14:18:55 InnoDB: Waiting for the background threads to start +161209 14:18:56 InnoDB: 5.5.53 started; log sequence number 1595675 +ERROR: 1050 Table 'plugin' already exists +161209 14:18:56 [ERROR] Aborting + +161209 14:18:56 InnoDB: Starting shutdown... +161209 14:18:56 InnoDB: Shutdown completed; log sequence number 1595675 +161209 14:18:56 [Note] /usr/sbin/mysqld: Shutdown complete + +161209 14:18:56 [Warning] Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead. +161209 14:18:56 [Note] Plugin 'FEDERATED' is disabled. +161209 14:18:56 InnoDB: The InnoDB memory heap is disabled +161209 14:18:56 InnoDB: Mutexes and rw_locks use GCC atomic builtins +161209 14:18:56 InnoDB: Compressed tables use zlib 1.2.3.4 +161209 14:18:56 InnoDB: Initializing buffer pool, size = 128.0M +161209 14:18:56 InnoDB: Completed initialization of buffer pool +161209 14:18:57 InnoDB: highest supported file format is Barracuda. +161209 14:18:57 InnoDB: Waiting for the background threads to start +161209 14:18:58 InnoDB: 5.5.53 started; log sequence number 1595675 +161209 14:18:58 [Note] Server hostname (bind-address): '127.0.0.1'; port: 3306 +161209 14:18:58 [Note] - '127.0.0.1' resolves to '127.0.0.1'; +161209 14:18:58 [Note] Server socket created on IP: '127.0.0.1'. +161209 14:18:58 [Note] Event Scheduler: Loaded 0 events +161209 14:18:58 [Note] /usr/sbin/mysqld: ready for connections. +Version: '5.5.53-0ubuntu0.12.04.1' socket: '/var/run/mysqld/mysqld.sock' port: 3306 (Ubuntu) +161209 14:37:57 [Note] /usr/sbin/mysqld: Normal shutdown + +161209 14:37:57 [Note] Event Scheduler: Purging the queue. 0 events +161209 14:37:57 InnoDB: Starting shutdown... +161209 14:37:57 InnoDB: Shutdown completed; log sequence number 1595685 +161209 14:37:57 [Note] /usr/sbin/mysqld: Shutdown complete + +161209 14:37:57 [Warning] Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead. +161209 14:37:57 [Note] Plugin 'FEDERATED' is disabled. +161209 14:37:57 InnoDB: The InnoDB memory heap is disabled +161209 14:37:57 InnoDB: Mutexes and rw_locks use GCC atomic builtins +161209 14:37:57 InnoDB: Compressed tables use zlib 1.2.3.4 +161209 14:37:57 InnoDB: Initializing buffer pool, size = 128.0M +161209 14:37:57 InnoDB: Completed initialization of buffer pool +161209 14:37:57 InnoDB: highest supported file format is Barracuda. +161209 14:37:57 InnoDB: Waiting for the background threads to start +161209 14:37:58 InnoDB: 5.5.53 started; log sequence number 1595685 +161209 14:37:58 [Note] Server hostname (bind-address): '127.0.0.1'; port: 3306 +161209 14:37:58 [Note] - '127.0.0.1' resolves to '127.0.0.1'; +161209 14:37:58 [Note] Server socket created on IP: '127.0.0.1'. +161209 14:37:58 [Note] Event Scheduler: Loaded 0 events +161209 14:37:58 [Note] /usr/sbin/mysqld: ready for connections. +Version: '5.5.53-0ubuntu0.12.04.1-log' socket: '/var/run/mysqld/mysqld.sock' port: 3306 (Ubuntu) diff --git a/filebeat/module/mysql/slowlog/_meta/fields.yml b/filebeat/module/mysql/slowlog/_meta/fields.yml new file mode 100644 index 00000000000..c785d9bd4b1 --- /dev/null +++ b/filebeat/module/mysql/slowlog/_meta/fields.yml @@ -0,0 +1,42 @@ +- name: slowlog + type: group + description: > + Contains fields from the MySQL slow logs. + fields: + - name: user + description: > + The MySQL user that created the query. + - name: host + description: > + The host from where the user that created the query logged in. + - name: ip + description: > + The IP address from where the user that created the query logged in. + - name: query_time.sec + type: float + description: > + The total time the query took, in seconds, as a floating point number. + - name: lock_time.sec + type: float + description: > + The amount of time the query waited for the lock to be available. The + value is in seconds, as a floating point number. + - name: rows_sent + type: long + description: > + The number of rows returned by the query. + - name: rows_examined + type: long + description: > + The number of rows scanned by the query. + - name: timestamp + type: long + description: > + The unix timestamp taken from the `SET timestamp` query. + - name: query + description: > + The slow query. + - name: id + type: long + description: > + The connection ID for the query. diff --git a/filebeat/module/mysql/slowlog/config/slowlog.yml b/filebeat/module/mysql/slowlog/config/slowlog.yml new file mode 100644 index 00000000000..e9380f56998 --- /dev/null +++ b/filebeat/module/mysql/slowlog/config/slowlog.yml @@ -0,0 +1,10 @@ +- input_type: log + paths: + - {{path}} + multiline: + pattern: "^# User@Host: " + negate: true + match: after + fields: + source_type: mysql-slowlog + pipeline_id: {{beat.pipeline_id}} diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json new file mode 100644 index 00000000000..f943ec68001 --- /dev/null +++ b/filebeat/module/mysql/slowlog/ingest/pipeline.json @@ -0,0 +1,39 @@ +{ + "description": "Pipeline for parsing MySQL slow logs.", + "processors": [{ + "grok": { + "field": "message", + "patterns":[ + "^# User@Host: %{USER:mysql.slowlog.user}(\\[[^\\]]+\\])? @ %{HOSTNAME:mysql.slowlog.host} \\[(IP:mysql.slowlog.ip)?\\](\\s*Id:\\s* %{NUMBER:mysql.slowlog.id})?\n# Query_time: %{NUMBER:mysql.slowlog.query_time.sec}\\s* Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec}\\s* Rows_sent: %{NUMBER:mysql.slowlog.rows_sent}\\s* Rows_examined: %{NUMBER:mysql.slowlog.rows_examined}\n(SET timestamp=%{NUMBER:mysql.slowlog.timestamp};\n)?%{GREEDYMULTILINE:mysql.slowlog.query}" + ], + "pattern_definitions" : { + "GREEDYMULTILINE" : "(.|\n)*" + }, + "ignore_missing": true + } + }, { + "remove":{ + "field": "message" + } + }, { + "date": { + "field": "mysql.slowlog.timestamp", + "target_field": "@timestamp", + "formats": ["UNIX"], + "ignore_failure": true + } + }, { + "gsub": { + "field": "mysql.slowlog.query", + "pattern": "\n# Time: [0-9]+ [0-9][0-9]:[0-9][0-9]:[0-9][0-9](\\.[0-9]+)?$", + "replacement": "", + "ignore_failure": true + } + }], + "on_failure" : [{ + "set" : { + "field" : "error", + "value" : "{{ _ingest.on_failure_message }}" + } + }] +} diff --git a/filebeat/module/mysql/slowlog/manifest.yml b/filebeat/module/mysql/slowlog/manifest.yml new file mode 100644 index 00000000000..b902c13ba2e --- /dev/null +++ b/filebeat/module/mysql/slowlog/manifest.yml @@ -0,0 +1,11 @@ +module_version: 1.0 + +vars: + path: + default: /var/log/mysql/mysql-slow.log + os.darwin: /usr/local/var/mysql/{{builtin.hostname}}-slow.log + os.windows: "c:/programdata/MySQL/MySQL Server*/mysql-slow.log" + +ingest_pipeline: ingest/pipeline.json +prospectors: + - config/slowlog.yml diff --git a/filebeat/module/mysql/slowlog/test/mysql-darwin-brew-5.7.10.log b/filebeat/module/mysql/slowlog/test/mysql-darwin-brew-5.7.10.log new file mode 100644 index 00000000000..419db21faa3 --- /dev/null +++ b/filebeat/module/mysql/slowlog/test/mysql-darwin-brew-5.7.10.log @@ -0,0 +1,8 @@ +/usr/local/Cellar/mysql/5.7.10/bin/mysqld, Version: 5.7.10 (Homebrew). started with: +Tcp port: 3306 Unix socket: /tmp/mysql.sock +Time Id Command Argument +# Time: 2016-12-12T11:54:16.601172Z +# User@Host: root[root] @ localhost [] Id: 2 +# Query_time: 11.004467 Lock_time: 0.000000 Rows_sent: 1 Rows_examined: 0 +SET timestamp=1481543656; +select sleep(11); diff --git a/filebeat/module/mysql/slowlog/test/mysql-ubuntu-5.5.53.log b/filebeat/module/mysql/slowlog/test/mysql-ubuntu-5.5.53.log new file mode 100644 index 00000000000..d33bbfd3aa0 --- /dev/null +++ b/filebeat/module/mysql/slowlog/test/mysql-ubuntu-5.5.53.log @@ -0,0 +1,66 @@ +/usr/sbin/mysqld, Version: 5.5.53-0ubuntu0.12.04.1-log ((Ubuntu)). started with: +Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock +Time Id Command Argument +# Time: 161209 14:37:59 +# User@Host: debian-sys-maint[debian-sys-maint] @ localhost [] +# Query_time: 0.000153 Lock_time: 0.000061 Rows_sent: 1 Rows_examined: 5 +SET timestamp=1481294279; +SELECT count(*) FROM mysql.user WHERE user='root' and password=''; +# User@Host: debian-sys-maint[debian-sys-maint] @ localhost [] +# Query_time: 0.002456 Lock_time: 0.000095 Rows_sent: 31 Rows_examined: 81 +SET timestamp=1481294279; +select concat('select count(*) into @discard from `', + TABLE_SCHEMA, '`.`', TABLE_NAME, '`') + from information_schema.TABLES where ENGINE='MyISAM'; +# User@Host: debian-sys-maint[debian-sys-maint] @ localhost [] +# Query_time: 0.006278 Lock_time: 0.000153 Rows_sent: 0 Rows_examined: 808 +SET timestamp=1481294279; +select count(*) into @discard from `information_schema`.`COLUMNS`; +# User@Host: debian-sys-maint[debian-sys-maint] @ localhost [] +# Query_time: 0.000262 Lock_time: 0.000204 Rows_sent: 0 Rows_examined: 0 +SET timestamp=1481294279; +select count(*) into @discard from `information_schema`.`EVENTS`; +# User@Host: debian-sys-maint[debian-sys-maint] @ localhost [] +# Query_time: 0.000323 Lock_time: 0.000241 Rows_sent: 0 Rows_examined: 0 +SET timestamp=1481294279; +select count(*) into @discard from `information_schema`.`PARAMETERS`; +# User@Host: debian-sys-maint[debian-sys-maint] @ localhost [] +# Query_time: 0.007084 Lock_time: 0.000148 Rows_sent: 0 Rows_examined: 81 +SET timestamp=1481294279; +select count(*) into @discard from `information_schema`.`PARTITIONS`; +# User@Host: debian-sys-maint[debian-sys-maint] @ localhost [] +# Query_time: 0.000277 Lock_time: 0.000135 Rows_sent: 0 Rows_examined: 23 +SET timestamp=1481294279; +select count(*) into @discard from `information_schema`.`PLUGINS`; +# User@Host: debian-sys-maint[debian-sys-maint] @ localhost [] +# Query_time: 0.000254 Lock_time: 0.000159 Rows_sent: 0 Rows_examined: 1 +SET timestamp=1481294279; +select count(*) into @discard from `information_schema`.`PROCESSLIST`; +# User@Host: debian-sys-maint[debian-sys-maint] @ localhost [] +# Query_time: 0.000297 Lock_time: 0.000229 Rows_sent: 0 Rows_examined: 0 +SET timestamp=1481294279; +select count(*) into @discard from `information_schema`.`ROUTINES`; +# User@Host: debian-sys-maint[debian-sys-maint] @ localhost [] +# Query_time: 0.001676 Lock_time: 0.000156 Rows_sent: 0 Rows_examined: 0 +SET timestamp=1481294279; +select count(*) into @discard from `information_schema`.`TRIGGERS`; +# User@Host: debian-sys-maint[debian-sys-maint] @ localhost [] +# Query_time: 0.008782 Lock_time: 0.001187 Rows_sent: 0 Rows_examined: 0 +SET timestamp=1481294279; +select count(*) into @discard from `information_schema`.`VIEWS`; +# Time: 161209 14:39:02 +# User@Host: root[root] @ localhost [] +# Query_time: 2.000268 Lock_time: 0.000000 Rows_sent: 1 Rows_examined: 0 +SET timestamp=1481294342; +select sleep(2); +# Time: 161209 14:39:23 +# User@Host: root[root] @ localhost [] +# Query_time: 0.000138 Lock_time: 0.000056 Rows_sent: 0 Rows_examined: 0 +use mysql; +SET timestamp=1481294363; +select * from general_log; +# Time: 161209 14:39:40 +# User@Host: root[root] @ localhost [] +# Query_time: 0.000159 Lock_time: 0.000059 Rows_sent: 5 Rows_examined: 5 +SET timestamp=1481294380; +select * from user; diff --git a/filebeat/module/nginx/access/ingest/no_plugins.json b/filebeat/module/nginx/access/ingest/no_plugins.json index 59d394d4364..5f611c47496 100644 --- a/filebeat/module/nginx/access/ingest/no_plugins.json +++ b/filebeat/module/nginx/access/ingest/no_plugins.json @@ -27,5 +27,11 @@ "remove": { "field": "nginx.access.time" } + }], + "on_failure" : [{ + "set" : { + "field" : "error", + "value" : "{{ _ingest.on_failure_message }}" + } }] } diff --git a/filebeat/module/nginx/access/ingest/with_plugins.json b/filebeat/module/nginx/access/ingest/with_plugins.json index 32ccda3329f..962250354ac 100644 --- a/filebeat/module/nginx/access/ingest/with_plugins.json +++ b/filebeat/module/nginx/access/ingest/with_plugins.json @@ -41,5 +41,11 @@ "field": "nginx.access.remote_ip", "target_field": "nginx.access.geoip" } + }], + "on_failure" : [{ + "set" : { + "field" : "error", + "value" : "{{ _ingest.on_failure_message }}" + } }] } diff --git a/filebeat/module/nginx/error/ingest/pipeline.json b/filebeat/module/nginx/error/ingest/pipeline.json index dc5a7e4138c..71646ccc3fa 100644 --- a/filebeat/module/nginx/error/ingest/pipeline.json +++ b/filebeat/module/nginx/error/ingest/pipeline.json @@ -5,7 +5,12 @@ "field": "message", "patterns": [ "%{DATA:nginx.error.time} \\[%{DATA:nginx.error.level}\\] %{NUMBER:nginx.error.pid}#%{NUMBER:nginx.error.tid}: (\\*%{NUMBER:nginx.error.connection_id} )?%{GREEDYDATA:nginx.error.message}" - ] + ], + "on_failure" : [{ + "set" : { + "field" : "error", + "value" : "The error pattern didn't match on this event." + } } },{ "remove":{