Add loki logs dashboard to MySQL mixin (optional)

mysqld-mixin/.lint

@@ -0,0 +1,7 @@
+exclusions:
+    template-job-rule:
+        reason: "Renamed to $prometheus_datasource as loki datasource also added"
+    template-datasource-rule:
+        reason: "Renamed to $prometheus_datasource as loki datasource also added"
+    panel-datasource-rule:
+        reason: "Renamed to $prometheus_datasource as loki datasource also added"

mysqld-mixin/README.md

@@ -1,23 +1,92 @@
 # MySQLd Mixin
 
 The MySQLd Mixin is a set of configurable, reusable, and extensible alerts and
-dashboards based on the metrics exported by the MySQLd Exporter. The mixin creates
+dashboards based on the metrics exported by the MySQLd Exporter and Loki logs (optional). The mixin also creates
 recording and alerting rules for Prometheus and suitable dashboard descriptions
 for Grafana.
 
-To use them, you need to have `mixtool` and `jsonnetfmt` installed. If you
+MySQL Overview:
+![screenshot-0](https://storage.googleapis.com/grafanalabs-integration-assets/mysql/screenshots/screenshot0.png)
+MySQL Logs from Loki(optional):
+![screenshot-1](https://storage.googleapis.com/grafanalabs-integration-assets/mysql/screenshots/screenshot1.png)
+
+## Generate config files
+
+You can manually generate dashboards, but first you should install some tools:
+
+```bash
+go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest
+go install github.com/google/go-jsonnet/cmd/jsonnet@latest
+# or in brew: brew install go-jsonnet
+```
+
+For linting and formatting, you would also need `mixtool` and `jsonnetfmt` installed. If you
 have a working Go development environment, it's easiest to run the following:
+
 ```bash
-$ go get github.com/monitoring-mixins/mixtool/cmd/mixtool
-$ go get github.com/google/go-jsonnet/cmd/jsonnetfmt
+go install github.com/monitoring-mixins/mixtool/cmd/mixtool@latest
+go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest
 ```
 
 You can then build the Prometheus rules files `alerts.yaml` and
 `rules.yaml` and a directory `dashboard_out` with the JSON dashboard files
+
 for Grafana:
 ```bash
 $ make build
 ```
 
-For more advanced uses of mixins, see
-https://github.com/monitoring-mixins/docs.
+## Loki Logs configuration
+
+To enable logs support in MySQLd mixin, enable them in config.libsonnet first:
+
+```
+{
+  _config+:: {
+    enableLokiLogs: true,
+  },
+}
+
+```
+
+then run
+```bash
+$ make build
+```
+
+This would generate MySQL logs dashboard, as well as modified MySQL overview dashboard.
+
+For proper logs correlation, you need to make sure that `job` and `instance` labels values match for both mysql_exporter metrics and logs, collected by [Promtail](https://grafana.com/docs/loki/latest/clients/promtail/) or [Grafana Agent](https://grafana.com/docs/grafana-cloud/agent/).
+
+To scrape MySQL logs the following promtail config snippet can be used for `job=integrations/mysql` and `instance=mysql-01`:
+
+```yaml
+scrape_configs:
+- job_name: integrations/mysql 
+  static_configs:
+    - labels:
+        instance: mysql-01 # must match instance used in mysqld_exporter
+        job: integrations/mysql # must match job used in mysqld_exporter
+        __path__: /var/log/mysql/*.log
+  pipeline_stages:
+    - 
+      # logs of mysql in sample-apps https://dev.mysql.com/doc/refman/8.0/en/error-log-format.html
+      # format time thread [label] [err_code] [subsystem] msg
+      # The [err_code] and [subsystem] fields were added in MySQL 8.0
+      # https://regex101.com/r/jwEke3/2
+      regex:
+          expression: '(?P<timestamp>.+) (?P<thread>[\d]+) \[(?P<label>.+?)\]( \[(?P<err_code>.+?)\] \[(?P<subsystem>.+?)\])? (?P<msg>.+)'
+    - labels:
+        label:
+        err_code:
+        subsystem:
+      # (optional) uncomment parse timestamp, but make sure you set the proper location to parse timezone
+      #- timestamp:
+      #    source: timestamp
+      #    fallback_formats: ["2006-01-02 15:04:05"]
+      #    format: "2006-01-02T15:04:05.000000Z"
+      #    location: Etc/UTC
+    - drop:
+        expression: "^ *$"
+        drop_counter_reason: "drop empty lines"
+```

mysqld-mixin/config.libsonnet

@@ -0,0 +1,11 @@
+{
+  _config+:: {
+    // Grafana dashboard IDs are necessary for stable links for dashboards
+    grafanaDashboardIDs: {
+      'mysql-overview.json': '549c2bf8936f7767ea6ac47c47b00f2a',
+      'mysql-logs.json': 'DlHAFwE7z',
+    },
+
+    enableLokiLogs: false,
+  },
+}

mysqld-mixin/dashboards/dashboards.libsonnet

@@ -0,0 +1,56 @@
+{
+
+  grafanaDashboards::
+    if $._config.enableLokiLogs then {
+      'mysql-overview.json':
+        (import 'mysql-overview.json')
+        +
+        {
+          links+: [
+            {
+              asDropdown: false,
+              icon: 'dashboard',
+              includeVars: true,
+              keepTime: true,
+              tags: [],
+              targetBlank: false,
+              title: 'MySQL Logs',
+              tooltip: '',
+              type: 'link',
+              url: 'd/%s' % $._config.grafanaDashboardIDs['mysql-logs.json'],
+            },
+          ],
+          uid: $._config.grafanaDashboardIDs['mysql-overview.json'],
+        },
+      'mysql-logs.json':
+        (import 'mysql-logs.json')
+        +
+        {
+
+          links+: [
+            {
+              asDropdown: false,
+              icon: 'dashboard',
+              includeVars: true,
+              keepTime: true,
+              tags: [],
+              targetBlank: false,
+              title: 'MySQL Overview',
+              tooltip: '',
+              type: 'link',
+              url: 'd/%s' % $._config.grafanaDashboardIDs['mysql-overview.json'],
+            },
+          ],
+
+
+          uid: $._config.grafanaDashboardIDs['mysql-logs.json'],
+
+        },
+    }
+    else {
+      'mysql-overview.json':
+        (import 'mysql-overview.json')
+        +
+        { uid: $._config.grafanaDashboardIDs['mysql-overview.json'] },
+    },
+}