Add datasource for each Loki instance

CHANGELOG.md

@@ -2,8 +2,12 @@
 ## Release 2.11.1 (in development)
 ### Enhancements
 
+- Add each Loki instance as datasources in Grafana dashboards
+  (PR[#3681](https://github.com/scality/metalk8s/pull/3681))
+
 - Bump Grafana image version to 8.3.4-ubuntu
   (PR[#3684](https://github.com/scality/metalk8s/pull/3684))
+
 ## Release 2.11.0
 ### Additions
 

buildchain/buildchain/salt_tree.py

@@ -334,6 +334,7 @@ def task(self) -> types.TaskDict:
     Path("salt/metalk8s/addons/logging/loki/deployed/datasource.sls"),
     Path("salt/metalk8s/addons/logging/loki/deployed/init.sls"),
     Path("salt/metalk8s/addons/logging/loki/deployed/loki-configuration-secret.sls"),
+    Path("salt/metalk8s/addons/logging/loki/deployed/service.sls"),
     Path("salt/metalk8s/addons/logging/loki/deployed/service-configuration.sls"),
     Path("salt/metalk8s/addons/logging/loki/post-upgrade.sls"),
     Path("salt/metalk8s/addons/prometheus-adapter/deployed/chart.sls"),

salt/metalk8s/addons/logging/deployed/files/logs.json

@@ -836,6 +836,23 @@
         "alignLevel": null
       }
     },
+    {
+      "datasource": null,
+      "description": "",
+      "gridPos": {
+        "h": 2,
+        "w": 24,
+        "x": 0,
+        "y": 7
+      },
+      "id": 44,
+      "options": {
+        "content": "<p style=\"color: orange; text-align: center;\"><b>You can select a specific Loki instance from the dropdown selector \"logs\" above, to force the instance queried.</b></p>",
+        "mode": "html"
+      },
+      "pluginVersion": "8.0.7",
+      "type": "text"
+    },
     {
       "datasource": "$logs",
       "gridPos": {

salt/metalk8s/addons/logging/deployed/files/loki.json

@@ -2007,6 +2007,23 @@
       ],
       "valueName": "current"
     },
+    {
+      "datasource": null,
+      "description": "",
+      "gridPos": {
+        "h": 2,
+        "w": 18,
+        "x": 6,
+        "y": 33
+      },
+      "id": 110,
+      "options": {
+        "content": "<p style=\"color: orange; text-align: center;\"><b>You can select a specific Loki instance from the dropdown selector \"logs\" above, to force the instance queried.</b></p>",
+        "mode": "html"
+      },
+      "pluginVersion": "8.0.7",
+      "type": "text"
+    },
     {
       "aliasColors": {},
       "bars": false,

salt/metalk8s/addons/logging/loki/deployed/datasource.sls

@@ -1,7 +1,16 @@
+{%- set loki_defaults = salt.slsutil.renderer(
+        'salt://metalk8s/addons/logging/loki/config/loki.yaml', saltenv=saltenv
+    )
+%}
+{%- set loki = salt.metalk8s_service_configuration.get_service_conf(
+        'metalk8s-logging', 'metalk8s-loki-config', loki_defaults
+    )
+%}
+
 include:
   - metalk8s.addons.prometheus-operator.deployed.namespace
 
-Deploy ConfigMap for Loki datasource:
+Deploy ConfigMap for Loki datasources:
   metalk8s_kubernetes.object_present:
     - manifest:
         apiVersion: v1
@@ -24,3 +33,11 @@ Deploy ConfigMap for Loki datasource:
               access: proxy
               url: http://loki.metalk8s-logging.svc:3100/
               version: 1
+{%- for index in range(loki.spec.deployment.replicas) %}
+            - name: Loki-{{ index }}
+              uid: metalk8s-loki-{{ index }}
+              type: loki
+              access: proxy
+              url: http://loki-{{ index }}.metalk8s-logging.svc:3100/
+              version: 1
+{%- endfor %}

salt/metalk8s/addons/logging/loki/deployed/init.sls

@@ -3,4 +3,5 @@ include:
   - .service-configuration
   - .loki-configuration-secret
   - .chart
+  - .service
   - .datasource

salt/metalk8s/addons/logging/loki/deployed/service.sls

@@ -0,0 +1,35 @@
+{%- set loki_defaults = salt.slsutil.renderer(
+        'salt://metalk8s/addons/logging/loki/config/loki.yaml', saltenv=saltenv
+    )
+%}
+{%- set loki = salt.metalk8s_service_configuration.get_service_conf(
+        'metalk8s-logging', 'metalk8s-loki-config', loki_defaults
+    )
+%}
+
+{%- for index in range(loki.spec.deployment.replicas) %}
+Deploy loki-{{ index }} service object:
+  metalk8s_kubernetes.object_present:
+    - manifest:
+        apiVersion: v1
+        kind: Service
+        metadata:
+          name: loki-{{ index }}
+          namespace: metalk8s-logging
+          labels:
+            app: loki
+            app.kubernetes.io/name: loki
+            app.kubernetes.io/part-of: metalk8s
+        spec:
+          clusterIP: None
+          ports:
+            - name: http-metrics
+              port: 3100
+              protocol: TCP
+              targetPort: http-metrics
+          selector:
+            app: loki
+            release: loki
+            statefulset.kubernetes.io/pod-name: loki-{{ index }}
+          type: ClusterIP
+{%- endfor %}

tests/post/features/logging.feature

@@ -25,3 +25,7 @@ Feature: Logging stack is up and running
     Given the Kubernetes API is available
     And the Loki API is available
     Then we can retrieve 'Watchdog' alert from Loki API
+
+  Scenario: We can access a specific Loki instance
+    Given the Kubernetes API is available
+    Then the Loki API is available through Service 'loki-0'

tests/post/steps/test_logging.py

@@ -57,20 +57,8 @@ def test_logging_pipeline_is_working(host):
 
 
 @given("the Loki API is available")
-def check_loki_api(k8s_client):
-    def _check_loki_ready():
-        # NOTE: We use Kubernetes client instead of DynamicClient as it
-        # ease the "service proxy path"
-        client = kubernetes.client.CoreV1Api(api_client=k8s_client.client)
-        try:
-            response = client.connect_get_namespaced_service_proxy_with_path(
-                "loki:http-metrics", "metalk8s-logging", path="ready"
-            )
-        except Exception as exc:  # pylint: disable=broad-except
-            assert False, str(exc)
-        assert response == "ready\n"
-
-    utils.retry(_check_loki_ready, times=10, wait=2, name="checking Loki API ready")
+def given_check_loki_api(k8s_client):
+    check_loki_api(k8s_client, "loki")
 
 
 @given("we have set up a logger pod", target_fixture="pod_creation_ts")
@@ -232,6 +220,11 @@ def _check_alert_exists():
     )
 
 
+@then("the Loki API is available through Service '{service}'")
+def then_check_loki_api(k8s_client, service):
+    check_loki_api(k8s_client, service)
+
+
 # }}}
 
 # Helpers {{{
@@ -260,4 +253,20 @@ def query_loki_api(k8s_client, content, route="query"):
     return response
 
 
+def check_loki_api(k8s_client, service):
+    def _check_loki_ready():
+        # NOTE: We use Kubernetes client instead of DynamicClient as it
+        # ease the "service proxy path"
+        client = kubernetes.client.CoreV1Api(api_client=k8s_client.client)
+        try:
+            response = client.connect_get_namespaced_service_proxy_with_path(
+                f"{service}:http-metrics", "metalk8s-logging", path="ready"
+            )
+        except Exception as exc:  # pylint: disable=broad-except
+            assert False, str(exc)
+        assert response == "ready\n"
+
+    utils.retry(_check_loki_ready, times=10, wait=2, name="checking Loki API ready")
+
+
 # }}}