From eea1f296754fd7eacae57190c8af9a3d06087b02 Mon Sep 17 00:00:00 2001 From: Vitaly Zhuravlev Date: Thu, 17 Feb 2022 17:40:43 +0300 Subject: [PATCH 01/11] add logs panel to mixin dashboard Signed-off-by: Vitaly Zhuravlev --- mysqld-mixin/dashboards/mysql-overview.json | 232 ++++++++++++++++++++ 1 file changed, 232 insertions(+) diff --git a/mysqld-mixin/dashboards/mysql-overview.json b/mysqld-mixin/dashboards/mysql-overview.json index c45363c5..baeed296 100644 --- a/mysqld-mixin/dashboards/mysql-overview.json +++ b/mysqld-mixin/dashboards/mysql-overview.json @@ -3644,7 +3644,239 @@ "align": false, "alignLevel": null } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 122 + }, + "id": 396, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "id": 18, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 129 + }, + "type": "timeseries", + "title": "Logs by Level", + "datasource": { + "uid": "${logs_datasource}", + "type": "loki" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "barAlignment": 0, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "auto", + "pointSize": 5, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic", + "fixedColor": "red" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "single" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [ + "sum" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${logs_datasource}" + }, + "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", label=~\".+\"}[$__interval])) by (label)", + "legendFormat": "{{ label }}", + "refId": "A" + } + ], + "interval": null + }, + { + "id": 399, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 129 + }, + "type": "timeseries", + "title": "Logs by Error Codes", + "datasource": { + "uid": "${logs_datasource}", + "type": "loki" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "barAlignment": 0, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "auto", + "pointSize": 5, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic", + "fixedColor": "red" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "single" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [ + "sum" + ] + } + }, + "targets": [ + { + "datasource": { + "uid": "${logs_datasource}", + "type": "loki" + }, + "refId": "B", + "hide": false, + "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", err_code!=\"\", label=~\".+\"}[$__interval])) by (label, err_code)", + "legendFormat": "{{ label }}: {{ err_code }}" + } + ], + "description": "Error codes are available in MySQL 8.0 and above.", + "interval": null + }, + { + "id": 398, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 135 + }, + "type": "logs", + "title": "MySQL Errors Log", + "datasource": { + "type": "loki", + "uid": "${logs_datasource}" + }, + "options": { + "showTime": false, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": false, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${logs_datasource}" + }, + "expr": "{job=~\"$job\", instance=~\"$instance\"}", + "refId": "A" + } + ] } + + ], "refresh": "10s", "schemaVersion": 25, From 8cb9fdb80dadb84d1ca8ee564cd2a7baa77c616f Mon Sep 17 00:00:00 2001 From: Vitaly Zhuravlev Date: Sat, 5 Mar 2022 16:44:42 +0300 Subject: [PATCH 02/11] rename logs_datasource to loki_datasource Signed-off-by: Vitaly Zhuravlev --- mysqld-mixin/dashboards/mysql-overview.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mysqld-mixin/dashboards/mysql-overview.json b/mysqld-mixin/dashboards/mysql-overview.json index baeed296..78e78594 100644 --- a/mysqld-mixin/dashboards/mysql-overview.json +++ b/mysqld-mixin/dashboards/mysql-overview.json @@ -3669,7 +3669,7 @@ "type": "timeseries", "title": "Logs by Level", "datasource": { - "uid": "${logs_datasource}", + "uid": "${loki_datasource}", "type": "loki" }, "fieldConfig": { @@ -3739,7 +3739,7 @@ { "datasource": { "type": "loki", - "uid": "${logs_datasource}" + "uid": "${loki_datasource}" }, "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", label=~\".+\"}[$__interval])) by (label)", "legendFormat": "{{ label }}", @@ -3759,7 +3759,7 @@ "type": "timeseries", "title": "Logs by Error Codes", "datasource": { - "uid": "${logs_datasource}", + "uid": "${loki_datasource}", "type": "loki" }, "fieldConfig": { @@ -3828,7 +3828,7 @@ "targets": [ { "datasource": { - "uid": "${logs_datasource}", + "uid": "${loki_datasource}", "type": "loki" }, "refId": "B", @@ -3852,7 +3852,7 @@ "title": "MySQL Errors Log", "datasource": { "type": "loki", - "uid": "${logs_datasource}" + "uid": "${loki_datasource}" }, "options": { "showTime": false, @@ -3868,7 +3868,7 @@ { "datasource": { "type": "loki", - "uid": "${logs_datasource}" + "uid": "${loki_datasource}" }, "expr": "{job=~\"$job\", instance=~\"$instance\"}", "refId": "A" From 02d5f7dae0701d266506afdd6e7af8c1819ff3ff Mon Sep 17 00:00:00 2001 From: Vitaly Zhuravlev Date: Sat, 5 Mar 2022 17:15:52 +0300 Subject: [PATCH 03/11] rename datasource to prometheus_datasource Signed-off-by: Vitaly Zhuravlev --- mysqld-mixin/dashboards/mysql-overview.json | 88 ++++++++++----------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/mysqld-mixin/dashboards/mysql-overview.json b/mysqld-mixin/dashboards/mysql-overview.json index 78e78594..9b5f4429 100644 --- a/mysqld-mixin/dashboards/mysql-overview.json +++ b/mysqld-mixin/dashboards/mysql-overview.json @@ -22,7 +22,7 @@ "panels": [ { "collapsed": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "gridPos": { "h": 1, "w": 24, @@ -37,7 +37,7 @@ }, { "cacheTimeout": null, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "description": "**Uptime**\n\nThe amount of time since the last restart of the MySQL server process.", "fieldConfig": { "defaults": { @@ -115,7 +115,7 @@ }, { "cacheTimeout": null, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "description": "**Current QPS**\n\nBased on the queries reported by MySQL's ``SHOW STATUS`` command, it is the number of statements executed by the server within the last second. This variable includes statements executed within stored programs, unlike the Questions variable. It does not count \n``COM_PING`` or ``COM_STATISTICS`` commands.", "fieldConfig": { "defaults": { @@ -199,7 +199,7 @@ }, { "cacheTimeout": null, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "description": "**InnoDB Buffer Pool Size**\n\nInnoDB maintains a storage area called the buffer pool for caching data and indexes in memory. Knowing how the InnoDB buffer pool works, and taking advantage of it to keep frequently accessed data in memory, is one of the most important aspects of MySQL tuning. The goal is to keep the working set in memory. In most cases, this should be between 60%-90% of available memory on a dedicated database host, but depends on many factors.", "fieldConfig": { "defaults": { @@ -283,7 +283,7 @@ }, { "collapsed": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "gridPos": { "h": 1, "w": 24, @@ -301,7 +301,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 0, "description": "**Max Connections** \n\nMax Connections is the maximum permitted number of simultaneous client connections. By default, this is 151. Increasing this value increases the number of file descriptors that mysqld requires. If the required number of descriptors are not available, the server reduces the value of Max Connections.\n\nmysqld actually permits Max Connections + 1 clients to connect. The extra connection is reserved for use by accounts that have the SUPER privilege, such as root.\n\nMax Used Connections is the maximum number of connections that have been in use simultaneously since the server started.\n\nConnections is the number of connection attempts (successful or not) to the MySQL server.", "editable": true, @@ -452,7 +452,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**MySQL Active Threads**\n\nThreads Connected is the number of open connections, while Threads Running is the number of threads not sleeping.", "editable": true, @@ -602,7 +602,7 @@ }, { "collapsed": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "gridPos": { "h": 1, "w": 24, @@ -620,7 +620,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**MySQL Questions**\n\nThe number of statements executed by the server. This includes only statements sent to the server by clients and not statements executed within stored programs, unlike the Queries used in the QPS calculation. \n\nThis variable does not count the following commands:\n* ``COM_PING``\n* ``COM_STATISTICS``\n* ``COM_STMT_PREPARE``\n* ``COM_STMT_CLOSE``\n* ``COM_STMT_RESET``", "editable": true, @@ -736,7 +736,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**MySQL Thread Cache**\n\nThe thread_cache_size variable sets how many threads the server should cache to reuse. When a client disconnects, the client's threads are put in the cache if the cache is not full. It is autosized in MySQL 5.6.8 and above (capped to 100). Requests for threads are satisfied by reusing threads taken from the cache if possible, and only when the cache is empty is a new thread created.\n\n* *Threads_created*: The number of threads created to handle connections.\n* *Threads_cached*: The number of threads in the thread cache.", "editable": true, @@ -879,7 +879,7 @@ }, { "collapsed": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "gridPos": { "h": 1, "w": 24, @@ -897,7 +897,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "editable": true, "error": false, @@ -1029,7 +1029,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**MySQL Select Types**\n\nAs with most relational databases, selecting based on indexes is more efficient than scanning an entire table's data. Here we see the counters for selects not done with indexes.\n\n* ***Select Scan*** is how many queries caused full table scans, in which all the data in the table had to be read and either discarded or returned.\n* ***Select Range*** is how many queries used a range scan, which means MySQL scanned all rows in a given range.\n* ***Select Full Join*** is the number of joins that are not joined on an index, this is usually a huge performance hit.", "editable": true, @@ -1190,7 +1190,7 @@ }, { "collapsed": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "gridPos": { "h": 1, "w": 24, @@ -1208,7 +1208,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**MySQL Sorts**\n\nDue to a query's structure, order, or other requirements, MySQL sorts the rows before returning them. For example, if a table is ordered 1 to 10 but you want the results reversed, MySQL then has to sort the rows to return 10 to 1.\n\nThis graph also shows when sorts had to scan a whole table or a given range of a table in order to return the results and which could not have been sorted via an index.", "editable": true, @@ -1358,7 +1358,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**MySQL Slow Queries**\n\nSlow queries are defined as queries being slower than the long_query_time setting. For example, if you have long_query_time set to 3, all queries that take longer than 3 seconds to complete will show on this graph.", "editable": true, @@ -1466,7 +1466,7 @@ }, { "collapsed": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "gridPos": { "h": 1, "w": 24, @@ -1484,7 +1484,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**Aborted Connections**\n\nWhen a given host connects to MySQL and the connection is interrupted in the middle (for example due to bad credentials), MySQL keeps that info in a system table (since 5.6 this table is exposed in performance_schema).\n\nIf the amount of failed requests without a successful connection reaches the value of max_connect_errors, mysqld assumes that something is wrong and blocks the host from further connection.\n\nTo allow connections from that host again, you need to issue the ``FLUSH HOSTS`` statement.", "editable": true, @@ -1609,7 +1609,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**Table Locks**\n\nMySQL takes a number of different locks for varying reasons. In this graph we see how many Table level locks MySQL has requested from the storage engine. In the case of InnoDB, many times the locks could actually be row locks as it only takes table level locks in a few specific cases.\n\nIt is most useful to compare Locks Immediate and Locks Waited. If Locks waited is rising, it means you have lock contention. Otherwise, Locks Immediate rising and falling is normal activity.", "editable": true, @@ -1729,7 +1729,7 @@ }, { "collapsed": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "gridPos": { "h": 1, "w": 24, @@ -1747,7 +1747,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**MySQL Network Traffic**\n\nHere we can see how much network traffic is generated by MySQL. Outbound is network traffic sent from MySQL and Inbound is network traffic MySQL has received.", "editable": true, @@ -1867,7 +1867,7 @@ }, { "collapsed": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "gridPos": { "h": 1, "w": 24, @@ -1885,7 +1885,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 0, "description": "***System Memory***: Total Memory for the system.\\\n***InnoDB Buffer Pool Data***: InnoDB maintains a storage area called the buffer pool for caching data and indexes in memory.\\\n***TokuDB Cache Size***: Similar in function to the InnoDB Buffer Pool, TokuDB will allocate 50% of the installed RAM for its own cache.\\\n***Key Buffer Size***: Index blocks for MYISAM tables are buffered and are shared by all threads. key_buffer_size is the size of the buffer used for index blocks.\\\n***Adaptive Hash Index Size***: When InnoDB notices that some index values are being accessed very frequently, it builds a hash index for them in memory on top of B-Tree indexes.\\\n ***Query Cache Size***: The query cache stores the text of a SELECT statement together with the corresponding result that was sent to the client. The query cache has huge scalability problems in that only one thread can do an operation in the query cache at the same time.\\\n***InnoDB Dictionary Size***: The data dictionary is InnoDB ‘s internal catalog of tables. InnoDB stores the data dictionary on disk, and loads entries into memory while the server is running.\\\n***InnoDB Log Buffer Size***: The MySQL InnoDB log buffer allows transactions to run without having to write the log to disk before the transactions commit.", "editable": true, @@ -2067,7 +2067,7 @@ }, { "collapsed": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "gridPos": { "h": 1, "w": 24, @@ -2085,7 +2085,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**Top Command Counters**\n\nThe Com_{{xxx}} statement counter variables indicate the number of times each xxx statement has been executed. There is one status variable for each type of statement. For example, Com_delete and Com_update count [``DELETE``](https://dev.mysql.com/doc/refman/5.7/en/delete.html) and [``UPDATE``](https://dev.mysql.com/doc/refman/5.7/en/update.html) statements, respectively. Com_delete_multi and Com_update_multi are similar but apply to [``DELETE``](https://dev.mysql.com/doc/refman/5.7/en/delete.html) and [``UPDATE``](https://dev.mysql.com/doc/refman/5.7/en/update.html) statements that use multiple-table syntax.", "editable": true, @@ -2203,7 +2203,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**MySQL Handlers**\n\nHandler statistics are internal statistics on how MySQL is selecting, updating, inserting, and modifying rows, tables, and indexes.\n\nThis is in fact the layer between the Storage Engine and MySQL.\n\n* `read_rnd_next` is incremented when the server performs a full table scan and this is a counter you don't really want to see with a high value.\n* `read_key` is incremented when a read is done with an index.\n* `read_next` is incremented when the storage engine is asked to 'read the next index entry'. A high value means a lot of index scans are being done.", "editable": true, @@ -2314,7 +2314,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "editable": true, "error": false, @@ -2423,7 +2423,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "editable": true, "error": false, @@ -2533,7 +2533,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "editable": true, "error": false, @@ -2639,7 +2639,7 @@ }, { "collapsed": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "gridPos": { "h": 1, "w": 24, @@ -2657,7 +2657,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**MySQL Query Cache Memory**\n\nThe query cache has huge scalability problems in that only one thread can do an operation in the query cache at the same time. This serialization is true not only for SELECTs, but also for INSERT/UPDATE/DELETE.\n\nThis also means that the larger the `query_cache_size` is set to, the slower those operations become. In concurrent environments, the MySQL Query Cache quickly becomes a contention point, decreasing performance. MariaDB and AWS Aurora have done work to try and eliminate the query cache contention in their flavors of MySQL, while MySQL 8.0 has eliminated the query cache feature.\n\nThe recommended settings for most environments is to set:\n ``query_cache_type=0``\n ``query_cache_size=0``\n\nNote that while you can dynamically change these values, to completely remove the contention point you have to restart the database.", "editable": true, @@ -2780,7 +2780,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**MySQL Query Cache Activity**\n\nThe query cache has huge scalability problems in that only one thread can do an operation in the query cache at the same time. This serialization is true not only for SELECTs, but also for INSERT/UPDATE/DELETE.\n\nThis also means that the larger the `query_cache_size` is set to, the slower those operations become. In concurrent environments, the MySQL Query Cache quickly becomes a contention point, decreasing performance. MariaDB and AWS Aurora have done work to try and eliminate the query cache contention in their flavors of MySQL, while MySQL 8.0 has eliminated the query cache feature.\n\nThe recommended settings for most environments is to set:\n``query_cache_type=0``\n``query_cache_size=0``\n\nNote that while you can dynamically change these values, to completely remove the contention point you have to restart the database.", "editable": true, @@ -2940,7 +2940,7 @@ }, { "collapsed": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "gridPos": { "h": 1, "w": 24, @@ -2958,7 +2958,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "editable": true, "error": false, @@ -3066,7 +3066,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "editable": true, "error": false, @@ -3191,7 +3191,7 @@ }, { "collapsed": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "gridPos": { "h": 1, "w": 24, @@ -3209,7 +3209,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**MySQL Table Open Cache Status**\n\nThe recommendation is to set the `table_open_cache_instances` to a loose correlation to virtual CPUs, keeping in mind that more instances means the cache is split more times. If you have a cache set to 500 but it has 10 instances, each cache will only have 50 cached.\n\nThe `table_definition_cache` and `table_open_cache` can be left as default as they are auto-sized MySQL 5.6 and above (ie: do not set them to any value).", "editable": true, @@ -3365,7 +3365,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**MySQL Open Tables**\n\nThe recommendation is to set the `table_open_cache_instances` to a loose correlation to virtual CPUs, keeping in mind that more instances means the cache is split more times. If you have a cache set to 500 but it has 10 instances, each cache will only have 50 cached.\n\nThe `table_definition_cache` and `table_open_cache` can be left as default as they are auto-sized MySQL 5.6 and above (ie: do not set them to any value).", "editable": true, @@ -3490,7 +3490,7 @@ }, { "collapsed": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "gridPos": { "h": 1, "w": 24, @@ -3508,7 +3508,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "decimals": 2, "description": "**MySQL Table Definition Cache**\n\nThe recommendation is to set the `table_open_cache_instances` to a loose correlation to virtual CPUs, keeping in mind that more instances means the cache is split more times. If you have a cache set to 500 but it has 10 instances, each cache will only have 50 cached.\n\nThe `table_definition_cache` and `table_open_cache` can be left as default as they are auto-sized MySQL 5.6 and above (ie: do not set them to any value).", "editable": true, @@ -3892,9 +3892,9 @@ }, "hide": 0, "includeAll": false, - "label": "Data Source", + "label": "Metrics Data Source", "multi": false, - "name": "datasource", + "name": "prometheus_datasource", "options": [], "query": "prometheus", "refresh": 1, @@ -3911,7 +3911,7 @@ "hosted-grafana/cloudsql-proxy-mysql-exporter" ] }, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "definition": "label_values(mysql_up, job)", "hide": 0, "includeAll": true, @@ -3940,7 +3940,7 @@ "$__all" ] }, - "datasource": "$datasource", + "datasource": "$prometheus_datasource", "definition": "label_values(mysql_up, instance)", "hide": 0, "includeAll": true, From 09217629edd7842e8df248dca1c5a98f616fdc9b Mon Sep 17 00:00:00 2001 From: Vitaly Zhuravlev Date: Sat, 5 Mar 2022 17:51:38 +0300 Subject: [PATCH 04/11] add lint exceptions related to $prometheus_datasource Signed-off-by: Vitaly Zhuravlev --- mysqld-mixin/.lint | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 mysqld-mixin/.lint diff --git a/mysqld-mixin/.lint b/mysqld-mixin/.lint new file mode 100644 index 00000000..ccbfe751 --- /dev/null +++ b/mysqld-mixin/.lint @@ -0,0 +1,7 @@ +exclusions: + template-job-rule: + reason: "Renamed to $prometheus_datasource as loki datasource also added" + template-datasource-rule: + reason: "Renamed to $prometheus_datasource as loki datasource also added" + panel-datasource-rule: + reason: "Renamed to $prometheus_datasource as loki datasource also added" \ No newline at end of file From 900b317c87fbfceb5572413ef7a7d205e811eb27 Mon Sep 17 00:00:00 2001 From: Vitaly Zhuravlev Date: Mon, 21 Mar 2022 14:41:52 +0400 Subject: [PATCH 05/11] add optional loki panels Signed-off-by: Vitaly Zhuravlev --- mysqld-mixin/.gitignore | 2 + mysqld-mixin/config.libsonnet | 5 + mysqld-mixin/dashboards/mysql-overview.json | 232 -------------------- mysqld-mixin/jsonnetfile.json | 15 ++ mysqld-mixin/lib/loki-panels.json | 232 ++++++++++++++++++++ mysqld-mixin/mixin.libsonnet | 29 ++- 6 files changed, 281 insertions(+), 234 deletions(-) create mode 100644 mysqld-mixin/config.libsonnet create mode 100644 mysqld-mixin/jsonnetfile.json create mode 100644 mysqld-mixin/lib/loki-panels.json diff --git a/mysqld-mixin/.gitignore b/mysqld-mixin/.gitignore index 97bf5f5c..ac3d221c 100644 --- a/mysqld-mixin/.gitignore +++ b/mysqld-mixin/.gitignore @@ -1,3 +1,5 @@ /alerts.yaml /rules.yaml dashboards_out +vendor +jsonnetfile.lock.json diff --git a/mysqld-mixin/config.libsonnet b/mysqld-mixin/config.libsonnet new file mode 100644 index 00000000..c21da6c5 --- /dev/null +++ b/mysqld-mixin/config.libsonnet @@ -0,0 +1,5 @@ +{ + _config+:: { + enableLokiLogs: false, + }, +} \ No newline at end of file diff --git a/mysqld-mixin/dashboards/mysql-overview.json b/mysqld-mixin/dashboards/mysql-overview.json index 9b5f4429..13e26ac6 100644 --- a/mysqld-mixin/dashboards/mysql-overview.json +++ b/mysqld-mixin/dashboards/mysql-overview.json @@ -3644,239 +3644,7 @@ "align": false, "alignLevel": null } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 122 - }, - "id": 396, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "id": 18, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 129 - }, - "type": "timeseries", - "title": "Logs by Level", - "datasource": { - "uid": "${loki_datasource}", - "type": "loki" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "lineInterpolation": "linear", - "barAlignment": 0, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "auto", - "pointSize": 5, - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - }, - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "color": { - "mode": "palette-classic", - "fixedColor": "red" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "single" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [ - "sum" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${loki_datasource}" - }, - "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", label=~\".+\"}[$__interval])) by (label)", - "legendFormat": "{{ label }}", - "refId": "A" - } - ], - "interval": null - }, - { - "id": 399, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 129 - }, - "type": "timeseries", - "title": "Logs by Error Codes", - "datasource": { - "uid": "${loki_datasource}", - "type": "loki" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "barAlignment": 0, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "auto", - "pointSize": 5, - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - }, - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "color": { - "mode": "palette-classic", - "fixedColor": "red" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "single" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [ - "sum" - ] - } - }, - "targets": [ - { - "datasource": { - "uid": "${loki_datasource}", - "type": "loki" - }, - "refId": "B", - "hide": false, - "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", err_code!=\"\", label=~\".+\"}[$__interval])) by (label, err_code)", - "legendFormat": "{{ label }}: {{ err_code }}" - } - ], - "description": "Error codes are available in MySQL 8.0 and above.", - "interval": null - }, - { - "id": 398, - "gridPos": { - "h": 20, - "w": 24, - "x": 0, - "y": 135 - }, - "type": "logs", - "title": "MySQL Errors Log", - "datasource": { - "type": "loki", - "uid": "${loki_datasource}" - }, - "options": { - "showTime": false, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": false, - "prettifyLogMessage": false, - "enableLogDetails": true, - "dedupStrategy": "none", - "sortOrder": "Descending" - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${loki_datasource}" - }, - "expr": "{job=~\"$job\", instance=~\"$instance\"}", - "refId": "A" - } - ] } - - ], "refresh": "10s", "schemaVersion": 25, diff --git a/mysqld-mixin/jsonnetfile.json b/mysqld-mixin/jsonnetfile.json new file mode 100644 index 00000000..0a577717 --- /dev/null +++ b/mysqld-mixin/jsonnetfile.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet-lib.git", + "subdir": "grafonnet" + } + }, + "version": "master" + } + ], + "legacyImports": false +} \ No newline at end of file diff --git a/mysqld-mixin/lib/loki-panels.json b/mysqld-mixin/lib/loki-panels.json new file mode 100644 index 00000000..3f141cdf --- /dev/null +++ b/mysqld-mixin/lib/loki-panels.json @@ -0,0 +1,232 @@ +[ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 122 + }, + "id": 990, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "id": 991, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 129 + }, + "type": "timeseries", + "title": "Logs by Level", + "datasource": { + "uid": "${loki_datasource}", + "type": "loki" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "barAlignment": 0, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "auto", + "pointSize": 5, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic", + "fixedColor": "red" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "single" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [ + "sum" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", label=~\".+\"}[$__interval])) by (label)", + "legendFormat": "{{ label }}", + "refId": "A" + } + ], + "interval": null + }, + { + "id": 992, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 129 + }, + "type": "timeseries", + "title": "Logs by Error Codes", + "datasource": { + "uid": "${loki_datasource}", + "type": "loki" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "barAlignment": 0, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "auto", + "pointSize": 5, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic", + "fixedColor": "red" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "single" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [ + "sum" + ] + } + }, + "targets": [ + { + "datasource": { + "uid": "${loki_datasource}", + "type": "loki" + }, + "refId": "B", + "hide": false, + "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", err_code!=\"\", label=~\".+\"}[$__interval])) by (label, err_code)", + "legendFormat": "{{ label }}: {{ err_code }}" + } + ], + "description": "Error codes are available in MySQL 8.0 and above.", + "interval": null + }, + { + "id": 993, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 135 + }, + "type": "logs", + "title": "MySQL Errors Log", + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "options": { + "showTime": false, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": false, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "expr": "{job=~\"$job\", instance=~\"$instance\"}", + "refId": "A" + } + ] + } +] \ No newline at end of file diff --git a/mysqld-mixin/mixin.libsonnet b/mysqld-mixin/mixin.libsonnet index 9515cdea..5cba5bf4 100644 --- a/mysqld-mixin/mixin.libsonnet +++ b/mysqld-mixin/mixin.libsonnet @@ -1,5 +1,30 @@ +local grafana = import 'github.com/grafana/grafonnet-lib/grafonnet/grafana.libsonnet'; + + + (import 'config.libsonnet') + { - grafanaDashboards: { + + + grafanaDashboards:: + if $._config.enableLokiLogs then { + 'mysql-overview.json': (import 'dashboards/mysql-overview.json') + + { + templating+: + { + list+: [ + grafana.template.new( + name="loki_datasource", + label="Logs Data Source", + query='loki', + refresh='load', + datasource='loki' + ) + { type: "datasource"} + ] + }, + panels+: import 'lib/loki-panels.json' + }, + } + else { 'mysql-overview.json': (import 'dashboards/mysql-overview.json'), }, @@ -14,4 +39,4 @@ prometheusAlerts+: importRules(importstr 'alerts/general.yaml') + importRules(importstr 'alerts/galera.yaml'), -} +} \ No newline at end of file From eeffa82ede7d980764391c9fa3947fe654c620d9 Mon Sep 17 00:00:00 2001 From: Vitaly Zhuravlev Date: Mon, 21 Mar 2022 15:11:24 +0400 Subject: [PATCH 06/11] move dashboards to seperate file Signed-off-by: Vitaly Zhuravlev --- mysqld-mixin/dashboards/dashboards.libsonnet | 27 +++++++++++++++++++ mysqld-mixin/mixin.libsonnet | 28 +------------------- 2 files changed, 28 insertions(+), 27 deletions(-) create mode 100644 mysqld-mixin/dashboards/dashboards.libsonnet diff --git a/mysqld-mixin/dashboards/dashboards.libsonnet b/mysqld-mixin/dashboards/dashboards.libsonnet new file mode 100644 index 00000000..fd367385 --- /dev/null +++ b/mysqld-mixin/dashboards/dashboards.libsonnet @@ -0,0 +1,27 @@ +local grafana = import 'github.com/grafana/grafonnet-lib/grafonnet/grafana.libsonnet'; + +{ + + grafanaDashboards:: + if $._config.enableLokiLogs then { + 'mysql-overview.json': (import 'mysql-overview.json') + + { + templating+: + { + list+: [ + grafana.template.new( + name="loki_datasource", + label="Logs Data Source", + query='loki', + refresh='load', + datasource='loki' + ) + { type: "datasource"} + ] + }, + panels+: import '../lib/loki-panels.json' + }, + } + else { + 'mysql-overview.json': (import 'mysql-overview.json'), + } +} \ No newline at end of file diff --git a/mysqld-mixin/mixin.libsonnet b/mysqld-mixin/mixin.libsonnet index 5cba5bf4..d0e0d930 100644 --- a/mysqld-mixin/mixin.libsonnet +++ b/mysqld-mixin/mixin.libsonnet @@ -1,33 +1,7 @@ -local grafana = import 'github.com/grafana/grafonnet-lib/grafonnet/grafana.libsonnet'; - - (import 'config.libsonnet') + + (import 'dashboards/dashboards.libsonnet') + { - - grafanaDashboards:: - if $._config.enableLokiLogs then { - 'mysql-overview.json': (import 'dashboards/mysql-overview.json') + - { - templating+: - { - list+: [ - grafana.template.new( - name="loki_datasource", - label="Logs Data Source", - query='loki', - refresh='load', - datasource='loki' - ) + { type: "datasource"} - ] - }, - panels+: import 'lib/loki-panels.json' - }, - } - else { - 'mysql-overview.json': (import 'dashboards/mysql-overview.json'), - }, - // Helper function to ensure that we don't override other rules, by forcing // the patching of the groups list, and not the overall rules object. local importRules(rules) = { From 18fb84d45c639cbc88eb14e2df829d3aa5efe058 Mon Sep 17 00:00:00 2001 From: Vitaly Zhuravlev Date: Mon, 21 Mar 2022 20:40:15 +0400 Subject: [PATCH 07/11] add logs dashboard Signed-off-by: Vitaly Zhuravlev --- mysqld-mixin/README.md | 47 +- mysqld-mixin/config.libsonnet | 2 +- mysqld-mixin/dashboards/dashboards.libsonnet | 51 ++- mysqld-mixin/dashboards/mysql-logs.json | 436 +++++++++++++++++++ mysqld-mixin/lib/loki-panels.json | 232 ---------- mysqld-mixin/mixin.libsonnet | 6 +- 6 files changed, 517 insertions(+), 257 deletions(-) create mode 100644 mysqld-mixin/dashboards/mysql-logs.json delete mode 100644 mysqld-mixin/lib/loki-panels.json diff --git a/mysqld-mixin/README.md b/mysqld-mixin/README.md index c23605a0..f613be83 100644 --- a/mysqld-mixin/README.md +++ b/mysqld-mixin/README.md @@ -1,23 +1,62 @@ # MySQLd Mixin The MySQLd Mixin is a set of configurable, reusable, and extensible alerts and -dashboards based on the metrics exported by the MySQLd Exporter. The mixin creates +dashboards based on the metrics exported by the MySQLd Exporter and Loki logs (optional). The mixin also creates recording and alerting rules for Prometheus and suitable dashboard descriptions for Grafana. -To use them, you need to have `mixtool` and `jsonnetfmt` installed. If you +MySQL Overview: +![screenshot-0](https://storage.googleapis.com/grafanalabs-integration-assets/mysql/screenshots/screenshot0.png) +MySQL Logs from Loki(optional): +![screenshot-1](https://storage.googleapis.com/grafanalabs-integration-assets/mysql/screenshots/screenshot1.png) + +## Generate config files + +You can manually generate dashboards, but first you should install some tools: + +```bash +go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest +go install github.com/google/go-jsonnet/cmd/jsonnet@latest +# or in brew: brew install go-jsonnet +``` + +For linting and formatting, you would also need `mixtool` and `jsonnetfmt` installed. If you have a working Go development environment, it's easiest to run the following: + ```bash -$ go get github.com/monitoring-mixins/mixtool/cmd/mixtool -$ go get github.com/google/go-jsonnet/cmd/jsonnetfmt +go install github.com/monitoring-mixins/mixtool/cmd/mixtool@latest +go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest ``` You can then build the Prometheus rules files `alerts.yaml` and `rules.yaml` and a directory `dashboard_out` with the JSON dashboard files + for Grafana: ```bash $ make build ``` +### Loki Logs configuration + +For proper logs correlation, you need to make sure that `job` and `instance` labels values match for both mysql_exporter metrics and logs, collected by promtail or grafana agent. + +To enable logs support in MySQLd mixin, enable them in config.libsonnet first: + +``` +{ + _config+:: { + enableLokiLogs: true, + }, +} + +``` + +then run +```bash +$ make build +``` + +This would generate mysql logs dashboard, as well as modified mysql overview dashboard. + For more advanced uses of mixins, see https://github.com/monitoring-mixins/docs. diff --git a/mysqld-mixin/config.libsonnet b/mysqld-mixin/config.libsonnet index c21da6c5..78e95f75 100644 --- a/mysqld-mixin/config.libsonnet +++ b/mysqld-mixin/config.libsonnet @@ -2,4 +2,4 @@ _config+:: { enableLokiLogs: false, }, -} \ No newline at end of file +} diff --git a/mysqld-mixin/dashboards/dashboards.libsonnet b/mysqld-mixin/dashboards/dashboards.libsonnet index fd367385..47a4166a 100644 --- a/mysqld-mixin/dashboards/dashboards.libsonnet +++ b/mysqld-mixin/dashboards/dashboards.libsonnet @@ -4,24 +4,41 @@ local grafana = import 'github.com/grafana/grafonnet-lib/grafonnet/grafana.libso grafanaDashboards:: if $._config.enableLokiLogs then { - 'mysql-overview.json': (import 'mysql-overview.json') + - { - templating+: + 'mysql-overview.json': + (import 'mysql-overview.json') + + { - list+: [ - grafana.template.new( - name="loki_datasource", - label="Logs Data Source", - query='loki', - refresh='load', - datasource='loki' - ) + { type: "datasource"} - ] + links+: [ + { + asDropdown: false, + icon: 'dashboard', + includeVars: true, + keepTime: true, + tags: [], + targetBlank: false, + title: 'MySQL Logs', + tooltip: '', + type: 'link', + url: 'd/DlHAFwE7z', + }, + ], + templating+: + { + + list+: [ + grafana.template.new( + name='loki_datasource', + label='Logs Data Source', + query='loki', + refresh='load', + datasource='loki' + ) + { type: 'datasource' }, + ], + }, }, - panels+: import '../lib/loki-panels.json' - }, + 'mysql-logs.json': (import 'mysql-logs.json'), } else { - 'mysql-overview.json': (import 'mysql-overview.json'), - } -} \ No newline at end of file + 'mysql-overview.json': (import 'mysql-overview.json'), + }, +} diff --git a/mysqld-mixin/dashboards/mysql-logs.json b/mysqld-mixin/dashboards/mysql-logs.json new file mode 100644 index 00000000..425743e0 --- /dev/null +++ b/mysqld-mixin/dashboards/mysql-logs.json @@ -0,0 +1,436 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 11323, + "graphTooltip": 1, + "id": 3, + "iteration": 1647879063849, + "links": [ + { + "asDropdown": false, + "icon": "dashboard", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "MySQL Overview", + "tooltip": "", + "type": "link", + "url": "d/549c2bf8936f7767ea6ac47c47b00f2a" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 990, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 991, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", label=~\".+\"}[$__interval])) by (label)", + "legendFormat": "{{ label }}", + "refId": "A" + } + ], + "title": "Logs by Level", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "description": "Error codes are available in MySQL 8.0 and above.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 992, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", err_code!=\"\", label=~\".+\"}[$__interval])) by (label, err_code)", + "hide": false, + "legendFormat": "{{ label }}: {{ err_code }}", + "refId": "B" + } + ], + "title": "Logs by Error Codes", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 993, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "expr": "{job=~\"$job\", instance=~\"$instance\"}", + "refId": "A" + } + ], + "title": "MySQL Errors Log", + "type": "logs" + } + ], + "refresh": "10s", + "schemaVersion": 34, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Cortex", + "value": "Cortex" + }, + "datasource": "loki", + "hide": 0, + "includeAll": false, + "label": "Metrics Data Source", + "multi": false, + "name": "prometheus_datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "datasource", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "Loki", + "value": "Loki" + }, + "datasource": "loki", + "hide": 0, + "includeAll": false, + "label": "Logs Data Source", + "multi": false, + "name": "loki_datasource", + "options": [], + "query": "loki", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "datasource", + "useTags": false + }, + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${prometheus_datasource}" + }, + "definition": "label_values(mysql_up, job)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(mysql_up, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": [ + "mysql.sample-apps.svc.cluster.local:3306" + ], + "value": [ + "mysql.sample-apps.svc.cluster.local:3306" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${prometheus_datasource}" + }, + "definition": "label_values(mysql_up, instance)", + "hide": 0, + "includeAll": true, + "label": false, + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(mysql_up, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "hidden": false, + "notice": false, + "now": true, + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "", + "title": "MySQL Logs", + "uid": "DlHAFwE7z", + "version": 6, + "weekStart": "" + } \ No newline at end of file diff --git a/mysqld-mixin/lib/loki-panels.json b/mysqld-mixin/lib/loki-panels.json deleted file mode 100644 index 3f141cdf..00000000 --- a/mysqld-mixin/lib/loki-panels.json +++ /dev/null @@ -1,232 +0,0 @@ -[ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 122 - }, - "id": 990, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "id": 991, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 129 - }, - "type": "timeseries", - "title": "Logs by Level", - "datasource": { - "uid": "${loki_datasource}", - "type": "loki" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "lineInterpolation": "linear", - "barAlignment": 0, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "auto", - "pointSize": 5, - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - }, - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "color": { - "mode": "palette-classic", - "fixedColor": "red" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "single" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [ - "sum" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${loki_datasource}" - }, - "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", label=~\".+\"}[$__interval])) by (label)", - "legendFormat": "{{ label }}", - "refId": "A" - } - ], - "interval": null - }, - { - "id": 992, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 129 - }, - "type": "timeseries", - "title": "Logs by Error Codes", - "datasource": { - "uid": "${loki_datasource}", - "type": "loki" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "barAlignment": 0, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "auto", - "pointSize": 5, - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - }, - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "color": { - "mode": "palette-classic", - "fixedColor": "red" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "single" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [ - "sum" - ] - } - }, - "targets": [ - { - "datasource": { - "uid": "${loki_datasource}", - "type": "loki" - }, - "refId": "B", - "hide": false, - "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", err_code!=\"\", label=~\".+\"}[$__interval])) by (label, err_code)", - "legendFormat": "{{ label }}: {{ err_code }}" - } - ], - "description": "Error codes are available in MySQL 8.0 and above.", - "interval": null - }, - { - "id": 993, - "gridPos": { - "h": 20, - "w": 24, - "x": 0, - "y": 135 - }, - "type": "logs", - "title": "MySQL Errors Log", - "datasource": { - "type": "loki", - "uid": "${loki_datasource}" - }, - "options": { - "showTime": false, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": false, - "prettifyLogMessage": false, - "enableLogDetails": true, - "dedupStrategy": "none", - "sortOrder": "Descending" - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${loki_datasource}" - }, - "expr": "{job=~\"$job\", instance=~\"$instance\"}", - "refId": "A" - } - ] - } -] \ No newline at end of file diff --git a/mysqld-mixin/mixin.libsonnet b/mysqld-mixin/mixin.libsonnet index d0e0d930..f39436b3 100644 --- a/mysqld-mixin/mixin.libsonnet +++ b/mysqld-mixin/mixin.libsonnet @@ -1,5 +1,5 @@ - (import 'config.libsonnet') + - (import 'dashboards/dashboards.libsonnet') + +(import 'config.libsonnet') + +(import 'dashboards/dashboards.libsonnet') + { // Helper function to ensure that we don't override other rules, by forcing @@ -13,4 +13,4 @@ prometheusAlerts+: importRules(importstr 'alerts/general.yaml') + importRules(importstr 'alerts/galera.yaml'), -} \ No newline at end of file +} From 4386b5fb8782f9a92211947d872de00a5dcbcb0f Mon Sep 17 00:00:00 2001 From: Vitaly Zhuravlev Date: Wed, 23 Mar 2022 22:06:29 +0400 Subject: [PATCH 08/11] remove templating patch Signed-off-by: Vitaly Zhuravlev --- mysqld-mixin/dashboards/dashboards.libsonnet | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/mysqld-mixin/dashboards/dashboards.libsonnet b/mysqld-mixin/dashboards/dashboards.libsonnet index 47a4166a..4daa0a71 100644 --- a/mysqld-mixin/dashboards/dashboards.libsonnet +++ b/mysqld-mixin/dashboards/dashboards.libsonnet @@ -22,19 +22,6 @@ local grafana = import 'github.com/grafana/grafonnet-lib/grafonnet/grafana.libso url: 'd/DlHAFwE7z', }, ], - templating+: - { - - list+: [ - grafana.template.new( - name='loki_datasource', - label='Logs Data Source', - query='loki', - refresh='load', - datasource='loki' - ) + { type: 'datasource' }, - ], - }, }, 'mysql-logs.json': (import 'mysql-logs.json'), } From 1cd457cc5b47919983c345af4eedad6fc390ea42 Mon Sep 17 00:00:00 2001 From: Vitaly Zhuravlev Date: Tue, 29 Mar 2022 15:30:20 +0400 Subject: [PATCH 09/11] add static dashboard ids Signed-off-by: Vitaly Zhuravlev --- mysqld-mixin/config.libsonnet | 6 ++++ mysqld-mixin/dashboards/dashboards.libsonnet | 33 ++++++++++++++++++-- mysqld-mixin/dashboards/mysql-logs.json | 15 --------- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/mysqld-mixin/config.libsonnet b/mysqld-mixin/config.libsonnet index 78e95f75..575d7f82 100644 --- a/mysqld-mixin/config.libsonnet +++ b/mysqld-mixin/config.libsonnet @@ -1,5 +1,11 @@ { _config+:: { + // Grafana dashboard IDs are necessary for stable links for dashboards + grafanaDashboardIDs: { + 'mysql-overview.json': '549c2bf8936f7767ea6ac47c47b00f2a', + 'mysql-logs.json': 'DlHAFwE7z', + }, + enableLokiLogs: false, }, } diff --git a/mysqld-mixin/dashboards/dashboards.libsonnet b/mysqld-mixin/dashboards/dashboards.libsonnet index 4daa0a71..e377c167 100644 --- a/mysqld-mixin/dashboards/dashboards.libsonnet +++ b/mysqld-mixin/dashboards/dashboards.libsonnet @@ -19,13 +19,40 @@ local grafana = import 'github.com/grafana/grafonnet-lib/grafonnet/grafana.libso title: 'MySQL Logs', tooltip: '', type: 'link', - url: 'd/DlHAFwE7z', + url: 'd/%s' % $._config.grafanaDashboardIDs['mysql-logs.json'], }, ], + uid: $._config.grafanaDashboardIDs['mysql-overview.json'], + }, + 'mysql-logs.json': + (import 'mysql-logs.json') + + + { + + links+: [ + { + asDropdown: false, + icon: 'dashboard', + includeVars: true, + keepTime: true, + tags: [], + targetBlank: false, + title: 'MySQL Overview', + tooltip: '', + type: 'link', + url: 'd/%s' % $._config.grafanaDashboardIDs['mysql-overview.json'], + }, + ], + + + uid: $._config.grafanaDashboardIDs['mysql-logs.json'], + }, - 'mysql-logs.json': (import 'mysql-logs.json'), } else { - 'mysql-overview.json': (import 'mysql-overview.json'), + 'mysql-overview.json': + (import 'mysql-overview.json') + + + { uid: $._config.grafanaDashboardIDs['mysql-overview.json'] }, }, } diff --git a/mysqld-mixin/dashboards/mysql-logs.json b/mysqld-mixin/dashboards/mysql-logs.json index 425743e0..ab951b96 100644 --- a/mysqld-mixin/dashboards/mysql-logs.json +++ b/mysqld-mixin/dashboards/mysql-logs.json @@ -25,20 +25,6 @@ "graphTooltip": 1, "id": 3, "iteration": 1647879063849, - "links": [ - { - "asDropdown": false, - "icon": "dashboard", - "includeVars": true, - "keepTime": true, - "tags": [], - "targetBlank": false, - "title": "MySQL Overview", - "tooltip": "", - "type": "link", - "url": "d/549c2bf8936f7767ea6ac47c47b00f2a" - } - ], "liveNow": false, "panels": [ { @@ -374,7 +360,6 @@ "definition": "label_values(mysql_up, instance)", "hide": 0, "includeAll": true, - "label": false, "multi": true, "name": "instance", "options": [], From 7f3367a4fe7c61b48ba217cbac22fbcf8ac188eb Mon Sep 17 00:00:00 2001 From: Vitaly Zhuravlev Date: Tue, 29 Mar 2022 18:37:25 +0400 Subject: [PATCH 10/11] add promtail snippet to README Signed-off-by: Vitaly Zhuravlev --- mysqld-mixin/README.md | 44 +++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/mysqld-mixin/README.md b/mysqld-mixin/README.md index f613be83..8621f2ac 100644 --- a/mysqld-mixin/README.md +++ b/mysqld-mixin/README.md @@ -36,9 +36,7 @@ for Grafana: $ make build ``` -### Loki Logs configuration - -For proper logs correlation, you need to make sure that `job` and `instance` labels values match for both mysql_exporter metrics and logs, collected by promtail or grafana agent. +## Loki Logs configuration To enable logs support in MySQLd mixin, enable them in config.libsonnet first: @@ -56,7 +54,39 @@ then run $ make build ``` -This would generate mysql logs dashboard, as well as modified mysql overview dashboard. - -For more advanced uses of mixins, see -https://github.com/monitoring-mixins/docs. +This would generate MySQL logs dashboard, as well as modified MySQL overview dashboard. + +For proper logs correlation, you need to make sure that `job` and `instance` labels values match for both mysql_exporter metrics and logs, collected by [Promtail](https://grafana.com/docs/loki/latest/clients/promtail/) or [Grafana Agent](https://grafana.com/docs/grafana-cloud/agent/). + +To scrape MySQL logs the following promtail config snippet can be used for `job=integrations/mysql` and `instance=mysql-01`: + +```yaml +scrape_configs: +- job_name: integrations/mysql + static_configs: + - labels: + instance: mysql-01 # must match instance used in mysqld_exporter + job: integrations/mysql # must match job used in mysqld_exporter + __path__: /var/log/mysql/*.log + pipeline_stages: + - + # logs of mysql in sample-apps https://dev.mysql.com/doc/refman/8.0/en/error-log-format.html + # format time thread [label] [err_code] [subsystem] msg + # The [err_code] and [subsystem] fields were added in MySQL 8.0 + # https://regex101.com/r/jwEke3/2 + regex: + expression: '(?P.+) (?P[\d]+) \[(?P