Use monitoring-mixins for dashboards and alerts

Monitoring-mixins ([1]) allow upstream collaboration on alerts and
grafana dashboards and convergence on monitoring best practices through
bundling mixins with the corresponding software.
Downstream customization of alerts and dashboards may be done using
jsonnet ([2]).
Usage of jsonnet-bundler ([3]) gives a clear reference of where
dashboards and alerts originated and which version is used through
`.src/jsonnetfile.json` and `.src/jsonnetfile.lock.json`.

[1]
https://monitoring.mixins.dev/

[2]
https://github.com/google/jsonnet

[3]
https://github.com/jsonnet-bundler/jsonnet-bundler

Signed-off-by: Jan Horstmann <[email protected]>

.gitignore

@@ -0,0 +1 @@
+.src/vendor

.src/Containerfile

@@ -0,0 +1,28 @@
+ARG DEBIAN_VERSION=bookworm
+FROM debian:${DEBIAN_VERSION}
+
+ARG JB_VERSION=v0.6.0
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY files/mixins.sh /mixins.sh
+
+WORKDIR /srv/.src
+CMD ["/mixins.sh"]
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+        ca-certificates \
+        curl \
+        git \
+        jsonnet \
+    && curl --location --output /usr/local/bin/jb https://github.com/jsonnet-bundler/jsonnet-bundler/releases/download/${JB_VERSION}/jb-linux-amd64 \
+    && chmod +x /usr/local/bin/jb \
+    && apt-get remove -y \
+        curl \
+    && apt-get clean \
+    && apt-get autoremove -y \
+    && rm -rf \
+      /var/lib/apt/lists/* \
+      /var/tmp/*

.src/Dockerfile

@@ -0,0 +1 @@
+Containerfile

.src/files/mixins.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -x
+set -o
+
+RULES_OUT="../prometheus"
+DASHBOARDS_OUT="../grafana/dashboards"
+
+jb install
+
+for FOLDER in $(find mixins/ -maxdepth 1 -mindepth 1 -type d); do
+    for FILE in "${FOLDER}"/*.libsonnet; do
+        NAME=$(basename ${FILE%%.libsonnet})
+        echo -en "---\n" > "${RULES_OUT}/${NAME}.rules"
+        jsonnet -J vendor -S -e 'std.manifestYamlDoc((import "'"${FILE}"'").prometheusAlerts)' >> "${RULES_OUT}/${NAME}.rules"
+        echo -en "---\n" >> "${RULES_OUT}/${NAME}.rules"
+        jsonnet -J vendor -S -e 'std.manifestYamlDoc((import "'"${FILE}"'").prometheusRules)' >> "${RULES_OUT}/${NAME}.rules"
+	jsonnet -J vendor -m "${DASHBOARDS_OUT}/$(basename ${FOLDER})" -e '(import "'"${FILE}"'").grafanaDashboards'
+    done
+done

.src/jsonnetfile.json

@@ -0,0 +1,42 @@
+{
+  "version": 1,
+  "dependencies": [
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/alertmanager.git",
+          "subdir": "doc/alertmanager-mixin"
+        }
+      },
+      "version": "main"
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/ceph/ceph.git",
+          "subdir": "monitoring/ceph-mixin"
+        }
+      },
+      "version": "main"
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/node_exporter.git",
+          "subdir": "docs/node-mixin"
+        }
+      },
+      "version": "master"
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/prometheus.git",
+          "subdir": "documentation/prometheus-mixin"
+        }
+      },
+      "version": "main"
+    }
+  ],
+  "legacyImports": true
+}

.src/jsonnetfile.lock.json

@@ -0,0 +1,76 @@
+{
+  "version": 1,
+  "dependencies": [
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/ceph/ceph.git",
+          "subdir": "monitoring/ceph-mixin"
+        }
+      },
+      "version": "20af41d716086a13009d3404acdee6c7c1444ae0",
+      "sum": "ZnyCIu25NBI6Q3Ru7QK1DHf7DBMEURSMQdEJXzCyIgA="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/grafonnet-lib.git",
+          "subdir": "grafonnet"
+        }
+      },
+      "version": "a1d61cce1da59c71409b99b5c7568511fec661ea",
+      "sum": "342u++/7rViR/zj2jeJOjshzglkZ1SY+hFNuyCBFMdc="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/grafonnet-lib.git",
+          "subdir": "grafonnet-7.0"
+        }
+      },
+      "version": "a1d61cce1da59c71409b99b5c7568511fec661ea",
+      "sum": "gCtR9s/4D5fxU9aKXg0Bru+/njZhA0YjLjPiASc61FM="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
+          "subdir": "grafana-builder"
+        }
+      },
+      "version": "3805ed5082bb7db66cd75badb57a673c965aa24c",
+      "sum": "yxqWcq/N3E/a/XreeU6EuE6X7kYPnG0AspAQFKOjASo="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/alertmanager.git",
+          "subdir": "doc/alertmanager-mixin"
+        }
+      },
+      "version": "4fcb18ee41ecaff4ee1816ad363dc63fdf3583b9",
+      "sum": "IpF46ZXsm+0wJJAPtAre8+yxTNZA57mBqGpBP/r7/kw="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/node_exporter.git",
+          "subdir": "docs/node-mixin"
+        }
+      },
+      "version": "e6a9cfbdcdaa21bf9676c6cd37bef8160227f423",
+      "sum": "cQCW+1N0Xae5yXecCWDK2oAlN0luBS/5GrwBYSlaFms="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/prometheus.git",
+          "subdir": "documentation/prometheus-mixin"
+        }
+      },
+      "version": "6ce21b83992430ba11e4961829b01a5c44974b6c",
+      "sum": "dYLcLzGH4yF3qB7OGC/7z4nqeTNjv42L7Q3BENU8XJI="
+    }
+  ],
+  "legacyImports": false
+}

.src/mixins/ceph/ceph.libsonnet

@@ -0,0 +1,7 @@
+local ceph = import"ceph-mixin/mixin.libsonnet";
+
+ceph {
+  prometheusRules+:: {},
+  prometheusAlerts+:: {},
+  grafanaDashboards+:: {}
+}

.src/mixins/infrastructure/alertmanager.libsonnet

@@ -0,0 +1,7 @@
+local alertmanager = import"alertmanager-mixin/mixin.libsonnet";
+
+alertmanager {
+  prometheusRules+:: {},
+  prometheusAlerts+:: {},
+  grafanaDashboards+:: {}
+}

.src/mixins/infrastructure/node.libsonnet

@@ -0,0 +1,7 @@
+local node = import"node-mixin/mixin.libsonnet";
+
+node {
+  prometheusRules+:: {},
+  prometheusAlerts+:: {},
+  grafanaDashboards+:: {}
+}

.src/mixins/infrastructure/prometheus.libsonnet

@@ -0,0 +1,7 @@
+local prometheus = import"prometheus-mixin/mixin.libsonnet";
+
+prometheus {
+  prometheusRules+:: {},
+  prometheusAlerts+:: {},
+  grafanaDashboards+:: {}
+}

README.md

@@ -3,5 +3,31 @@
 Repository for Grafana dashboards and Prometheus alerting rules.
 For use with the Prometheus exporters from Kolla.
 
-The alerts here build on those from:
-https://github.com/samber/awesome-prometheus-alerts
+## Usage
+
+While it is possible to directly place `.rules` files into the `prometheus/` folder  and grafana dashboards into `grafana/dashboards` or a subfolder thereof, use of [monitoring-mixins](https://monitoring.mixins.dev) is encouraged.
+
+### Build the jsonnet container
+
+Build the `mixin` container to manage mixins
+
+```
+podman build -t mixins .src
+```
+
+### Adding/Updating mixins
+
+[jsonnet-bundler](https://github.com/jsonnet-bundler/jsonnet-bundler) is used to fetch mixins and their dependencies from upstream.
+
+* Add any new upstream mixins to `.src/jsonnetfile.json
+* Update the jsonnet-bundler dependencies in `.src/jsonnetfile.lock.json
+  ```
+  podman run -it -v .:/srv localhost/mixins:latest jb update
+  ```
+* Add a local libsonnet for any new mixins in `.src/mixins/$SUBFOLDER/$NAME.libsonnet
+* Apply any [customizations](https://monitoring.mixins.dev/#customising-the-mixin) to the local `$NAME.libsonnet`
+* Build alerts, rules and dashboards
+  ```
+  podman run -it -v .:/srv localhost/mixins:latest
+  ```
+* Check all changes into git, including the `.src/jsonnetfile.lock.json` and the generated alerts, rules and dashboards