From abc293a7d78c3a1a83db9dfcfe5d1624cab8b2ab Mon Sep 17 00:00:00 2001 From: Alexandre Allard Date: Mon, 24 Jan 2022 18:40:50 +0100 Subject: [PATCH 1/5] salt,logs: Deploy a Service per Loki Pod These services allow to communicate which each Pod individually, this is needed to deploy datasources to target a specific Loki instance. --- buildchain/buildchain/salt_tree.py | 1 + .../addons/logging/loki/deployed/init.sls | 1 + .../addons/logging/loki/deployed/service.sls | 35 +++++++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 salt/metalk8s/addons/logging/loki/deployed/service.sls diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py index 99f3b44abf..f105aff10f 100644 --- a/buildchain/buildchain/salt_tree.py +++ b/buildchain/buildchain/salt_tree.py @@ -334,6 +334,7 @@ def task(self) -> types.TaskDict: Path("salt/metalk8s/addons/logging/loki/deployed/datasource.sls"), Path("salt/metalk8s/addons/logging/loki/deployed/init.sls"), Path("salt/metalk8s/addons/logging/loki/deployed/loki-configuration-secret.sls"), + Path("salt/metalk8s/addons/logging/loki/deployed/service.sls"), Path("salt/metalk8s/addons/logging/loki/deployed/service-configuration.sls"), Path("salt/metalk8s/addons/logging/loki/post-upgrade.sls"), Path("salt/metalk8s/addons/prometheus-adapter/deployed/chart.sls"), diff --git a/salt/metalk8s/addons/logging/loki/deployed/init.sls b/salt/metalk8s/addons/logging/loki/deployed/init.sls index d5d2303535..ae37b7f74d 100644 --- a/salt/metalk8s/addons/logging/loki/deployed/init.sls +++ b/salt/metalk8s/addons/logging/loki/deployed/init.sls @@ -3,4 +3,5 @@ include: - .service-configuration - .loki-configuration-secret - .chart + - .service - .datasource diff --git a/salt/metalk8s/addons/logging/loki/deployed/service.sls b/salt/metalk8s/addons/logging/loki/deployed/service.sls new file mode 100644 index 0000000000..0ce14bfcaf --- /dev/null +++ b/salt/metalk8s/addons/logging/loki/deployed/service.sls @@ -0,0 +1,35 @@ +{%- set loki_defaults = salt.slsutil.renderer( + 'salt://metalk8s/addons/logging/loki/config/loki.yaml', saltenv=saltenv + ) +%} +{%- set loki = salt.metalk8s_service_configuration.get_service_conf( + 'metalk8s-logging', 'metalk8s-loki-config', loki_defaults + ) +%} + +{%- for index in range(loki.spec.deployment.replicas) %} +Deploy loki-{{ index }} service object: + metalk8s_kubernetes.object_present: + - manifest: + apiVersion: v1 + kind: Service + metadata: + name: loki-{{ index }} + namespace: metalk8s-logging + labels: + app: loki + app.kubernetes.io/name: loki + app.kubernetes.io/part-of: metalk8s + spec: + clusterIP: None + ports: + - name: http-metrics + port: 3100 + protocol: TCP + targetPort: http-metrics + selector: + app: loki + release: loki + statefulset.kubernetes.io/pod-name: loki-{{ index }} + type: ClusterIP +{%- endfor %} From 58a21bfef8962aeee121574568fccc6113e95a64 Mon Sep 17 00:00:00 2001 From: Alexandre Allard Date: Mon, 24 Jan 2022 18:42:04 +0100 Subject: [PATCH 2/5] salt,logs: Deploy a Grafana datasource per Loki instance This is a workaround to allow user to access a specific Loki instance in case one would not have all the logs (disk loss or node down for a long period of time). This way the user can select a healthy instance having all the data. --- .../logging/loki/deployed/datasource.sls | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/salt/metalk8s/addons/logging/loki/deployed/datasource.sls b/salt/metalk8s/addons/logging/loki/deployed/datasource.sls index 64d27665a0..93e9e5867d 100644 --- a/salt/metalk8s/addons/logging/loki/deployed/datasource.sls +++ b/salt/metalk8s/addons/logging/loki/deployed/datasource.sls @@ -1,7 +1,16 @@ +{%- set loki_defaults = salt.slsutil.renderer( + 'salt://metalk8s/addons/logging/loki/config/loki.yaml', saltenv=saltenv + ) +%} +{%- set loki = salt.metalk8s_service_configuration.get_service_conf( + 'metalk8s-logging', 'metalk8s-loki-config', loki_defaults + ) +%} + include: - metalk8s.addons.prometheus-operator.deployed.namespace -Deploy ConfigMap for Loki datasource: +Deploy ConfigMap for Loki datasources: metalk8s_kubernetes.object_present: - manifest: apiVersion: v1 @@ -24,3 +33,11 @@ Deploy ConfigMap for Loki datasource: access: proxy url: http://loki.metalk8s-logging.svc:3100/ version: 1 +{%- for index in range(loki.spec.deployment.replicas) %} + - name: Loki-{{ index }} + uid: metalk8s-loki-{{ index }} + type: loki + access: proxy + url: http://loki-{{ index }}.metalk8s-logging.svc:3100/ + version: 1 +{%- endfor %} From f3951e207b7d7ca2eede3e792e3415d0d7eaf8dc Mon Sep 17 00:00:00 2001 From: Alexandre Allard Date: Tue, 25 Jan 2022 09:52:55 +0100 Subject: [PATCH 3/5] salt,logs: Tell user that datasource can be changed Display a message on Loki and Logs Grafana dashboards to tell the user he can change the datasource to target a specific Loki instance --- .../addons/logging/deployed/files/logs.json | 17 +++++++++++++++++ .../addons/logging/deployed/files/loki.json | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/salt/metalk8s/addons/logging/deployed/files/logs.json b/salt/metalk8s/addons/logging/deployed/files/logs.json index bfb4e61306..1d0e4883b4 100644 --- a/salt/metalk8s/addons/logging/deployed/files/logs.json +++ b/salt/metalk8s/addons/logging/deployed/files/logs.json @@ -836,6 +836,23 @@ "alignLevel": null } }, + { + "datasource": null, + "description": "", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 44, + "options": { + "content": "

You can select a specific Loki instance from the dropdown selector \"logs\" above, to force the instance queried.

", + "mode": "html" + }, + "pluginVersion": "8.0.7", + "type": "text" + }, { "datasource": "$logs", "gridPos": { diff --git a/salt/metalk8s/addons/logging/deployed/files/loki.json b/salt/metalk8s/addons/logging/deployed/files/loki.json index 810b0a8755..3b069bebee 100644 --- a/salt/metalk8s/addons/logging/deployed/files/loki.json +++ b/salt/metalk8s/addons/logging/deployed/files/loki.json @@ -2007,6 +2007,23 @@ ], "valueName": "current" }, + { + "datasource": null, + "description": "", + "gridPos": { + "h": 2, + "w": 18, + "x": 6, + "y": 33 + }, + "id": 110, + "options": { + "content": "

You can select a specific Loki instance from the dropdown selector \"logs\" above, to force the instance queried.

", + "mode": "html" + }, + "pluginVersion": "8.0.7", + "type": "text" + }, { "aliasColors": {}, "bars": false, From 11e7270f5f73f08115b4a6a2f9dc996e12183434 Mon Sep 17 00:00:00 2001 From: Alexandre Allard Date: Tue, 25 Jan 2022 18:13:28 +0100 Subject: [PATCH 4/5] Add a CHANGELOG entry for Loki datasources --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c94b31711e..aa7769b900 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,12 @@ ## Release 2.11.1 (in development) ### Enhancements +- Add each Loki instance as datasources in Grafana dashboards + (PR[#3681](https://github.com/scality/metalk8s/pull/3681)) + - Bump Grafana image version to 8.3.4-ubuntu (PR[#3684](https://github.com/scality/metalk8s/pull/3684)) + ## Release 2.11.0 ### Additions From a1c93eb3e28592ee3c4b2e2563ef1b66722a1bee Mon Sep 17 00:00:00 2001 From: Alexandre Allard Date: Fri, 28 Jan 2022 16:32:53 +0100 Subject: [PATCH 5/5] tests: Add a test to ensure we can access Loki API This test is to ensure we can access a specific Loki instance through the Services. --- tests/post/features/logging.feature | 4 ++++ tests/post/steps/test_logging.py | 37 ++++++++++++++++++----------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/tests/post/features/logging.feature b/tests/post/features/logging.feature index 22cf38c50d..0d6d177521 100644 --- a/tests/post/features/logging.feature +++ b/tests/post/features/logging.feature @@ -25,3 +25,7 @@ Feature: Logging stack is up and running Given the Kubernetes API is available And the Loki API is available Then we can retrieve 'Watchdog' alert from Loki API + + Scenario: We can access a specific Loki instance + Given the Kubernetes API is available + Then the Loki API is available through Service 'loki-0' diff --git a/tests/post/steps/test_logging.py b/tests/post/steps/test_logging.py index 2fa1b531d0..b987860cfe 100644 --- a/tests/post/steps/test_logging.py +++ b/tests/post/steps/test_logging.py @@ -57,20 +57,8 @@ def test_logging_pipeline_is_working(host): @given("the Loki API is available") -def check_loki_api(k8s_client): - def _check_loki_ready(): - # NOTE: We use Kubernetes client instead of DynamicClient as it - # ease the "service proxy path" - client = kubernetes.client.CoreV1Api(api_client=k8s_client.client) - try: - response = client.connect_get_namespaced_service_proxy_with_path( - "loki:http-metrics", "metalk8s-logging", path="ready" - ) - except Exception as exc: # pylint: disable=broad-except - assert False, str(exc) - assert response == "ready\n" - - utils.retry(_check_loki_ready, times=10, wait=2, name="checking Loki API ready") +def given_check_loki_api(k8s_client): + check_loki_api(k8s_client, "loki") @given("we have set up a logger pod", target_fixture="pod_creation_ts") @@ -232,6 +220,11 @@ def _check_alert_exists(): ) +@then("the Loki API is available through Service '{service}'") +def then_check_loki_api(k8s_client, service): + check_loki_api(k8s_client, service) + + # }}} # Helpers {{{ @@ -260,4 +253,20 @@ def query_loki_api(k8s_client, content, route="query"): return response +def check_loki_api(k8s_client, service): + def _check_loki_ready(): + # NOTE: We use Kubernetes client instead of DynamicClient as it + # ease the "service proxy path" + client = kubernetes.client.CoreV1Api(api_client=k8s_client.client) + try: + response = client.connect_get_namespaced_service_proxy_with_path( + f"{service}:http-metrics", "metalk8s-logging", path="ready" + ) + except Exception as exc: # pylint: disable=broad-except + assert False, str(exc) + assert response == "ready\n" + + utils.retry(_check_loki_ready, times=10, wait=2, name="checking Loki API ready") + + # }}}