From a0acb6b749bc931da56d267354bdbf80d10110bf Mon Sep 17 00:00:00 2001 From: Alexandre Allard Date: Tue, 11 Aug 2020 15:37:32 +0200 Subject: [PATCH] tests: add scenario for logging pipeline spawn a container that logs a single line then try to retrieve it from Loki API Refs: #2701 --- tests/post/features/logging.feature | 7 ++ tests/post/steps/files/logger-pod.yaml.tpl | 20 +++ tests/post/steps/test_logging.py | 136 ++++++++++++++++++--- 3 files changed, 147 insertions(+), 16 deletions(-) create mode 100644 tests/post/steps/files/logger-pod.yaml.tpl diff --git a/tests/post/features/logging.feature b/tests/post/features/logging.feature index ed464619b4..e1abcf6e86 100644 --- a/tests/post/features/logging.feature +++ b/tests/post/features/logging.feature @@ -7,9 +7,16 @@ Feature: Logging stack is up and running Scenario: Expected Pods Given the Kubernetes API is available Then we have 1 running pod labeled 'app=loki' in namespace 'metalk8s-logging' + And we have 1 running pod labeled 'app=fluent-bit' in namespace 'metalk8s-logging' on node 'bootstrap' Scenario: Pushing log to Loki directly Given the Kubernetes API is available And the Loki API is available When we push an example log to Loki Then we can query this example log from Loki + + Scenario: Logging pipeline is working + Given the Kubernetes API is available + And the Loki API is available + And we have set up a 'logger-test' pod + Then we can retrieve logs from 'logger-test' pod in Loki API diff --git a/tests/post/steps/files/logger-pod.yaml.tpl b/tests/post/steps/files/logger-pod.yaml.tpl new file mode 100644 index 0000000000..c00dfb3873 --- /dev/null +++ b/tests/post/steps/files/logger-pod.yaml.tpl @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: $name + namespace: metalk8s-logging +spec: + tolerations: + - key: "node-role.kubernetes.io/bootstrap" + operator: "Equal" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/infra" + operator: "Equal" + effect: "NoSchedule" + containers: + - name: $name + image: $image + command: + - echo + - logging pipeline is working + restartPolicy: Never diff --git a/tests/post/steps/test_logging.py b/tests/post/steps/test_logging.py index 10725a5229..98c9e90da3 100644 --- a/tests/post/steps/test_logging.py +++ b/tests/post/steps/test_logging.py @@ -1,14 +1,26 @@ -import json +import pathlib +import string import time import uuid import pytest -from pytest_bdd import scenario, given, when, then +from pytest_bdd import scenario, given, when, then, parsers -from tests import utils +from tests import utils, kube_utils + +# Constants {{{ + +MANIFESTS_PATH = pathlib.Path("/etc/kubernetes/manifests/") +TEMPLATES_PATH = pathlib.Path("/srv/scality/_templates/") +LOGGER_POD_TEMPLATE = ( + pathlib.Path(__file__) / ".." / "files" / "logger-pod.yaml.tpl" +).resolve() + +# }}} # Fixtures {{{ + @pytest.fixture(scope='function') def context(): return {} @@ -33,6 +45,11 @@ def test_push_log_to_loki(host): pass +@scenario('../features/logging.feature', 'Logging pipeline is working') +def test_logging_pipeline_is_working(host): + pass + + # }}} # Given {{{ @@ -56,6 +73,60 @@ def _check_loki_ready(): ) +@given(parsers.parse("we have set up a '{pod_name}' pod"), + target_fixture='pod_creation_ts') +def set_up_logger_pod(host, nodename, k8s_client, utils_image, pod_name): + manifest_template = LOGGER_POD_TEMPLATE.read_text(encoding="utf-8") + manifest = string.Template(manifest_template).substitute( + name=pod_name, + image=utils_image, + ) + manifest_path = str(MANIFESTS_PATH / "{}.yaml".format(pod_name)) + template_path = str( + TEMPLATES_PATH / "{}.yaml".format(pod_name) + ) + + with host.sudo(): + host.run_test("mkdir -p %s", str(TEMPLATES_PATH)) + + write_template = utils.write_string(host, template_path, manifest) + assert write_template.rc == 0, ( + "Failed to create logger Pod manifest template '{}': {}" + ).format(template_path, write_template.stderr) + + pod_creation_ts = int(time.time()) + manage_static_pod = host.run( + "salt-call state.single metalk8s.static_pod_managed " + 'name=%s source=%s', + manifest_path, + template_path, + ) + + assert manage_static_pod.rc == 0, ( + "Failed to manage logger Pod with Salt: {}" + ).format(manage_static_pod.stderr) + + pod_fullname = "{}-{}".format(pod_name, nodename) + utils.retry( + kube_utils.check_pod_status( + k8s_client, pod_fullname, + namespace="metalk8s-logging", state="Succeeded", + ), + times=10, + wait=5, + name="wait for Pod '{}'".format(pod_fullname), + ) + + yield pod_creation_ts + + with host.sudo(): + for filename in (manifest_path, template_path): + cmd = host.run("rm -f %s", filename) + assert cmd.rc == 0, "Failed to remove file '{}': {}".format( + filename, cmd.stderr + ) + + # }}} # When {{{ @@ -102,21 +173,61 @@ def push_log_to_loki(k8s_client, context): @then("we can query this example log from Loki") def query_log_from_loki(k8s_client, context): + query = {'query': '{{identifier="{0}"}}'.format(context['test_log_id'])} + response = query_loki_api(k8s_client, query) + result_data = response[0]['data']['result'] + + assert result_data, \ + 'No test log found in Loki with identifier={}'.format( + context['test_log_id'] + ) + assert result_data[0]['stream']['identifier'] == context['test_log_id'] + + +@then(parsers.parse("we can retrieve logs from '{pod_name}' pod in Loki API")) +def retrieve_pod_logs_from_loki(k8s_client, nodename, pod_name, + pod_creation_ts): + pod_fullname = "{}-{}".format(pod_name, nodename) + query = { + 'query': '{{instance="{0}"}}'.format(pod_fullname), + 'start': pod_creation_ts, + } + + def _check_log_line_exists(): + response = query_loki_api(k8s_client, query, route='query_range') + try: + result_data = response[0]['data']['result'][0]['values'] + except IndexError: + result_data = [] + assert any("logging pipeline is working" in v[1] + for v in result_data), \ + 'No log found in Loki for pod "{}"'.format(pod_fullname) + + utils.retry( + _check_log_line_exists, + times=20, + wait=3, + name="check that a log exists for pod '{0}'".format(pod_fullname) + ) + +# }}} + +# Helpers {{{ + + +def query_loki_api(k8s_client, content, route='query'): # With current k8s client we cannot pass query_params so we need to # use `call_api` directly path_params = { 'name': 'loki:http-metrics', 'namespace': 'metalk8s-logging', - 'path': 'loki/api/v1/query' - } - query_params = { - 'query': '{identifier="' + context['test_log_id'] + '"}' + 'path': 'loki/api/v1/{0}'.format(route) } response = k8s_client.api_client.call_api( '/api/v1/namespaces/{namespace}/services/{name}/proxy/{path}', 'GET', path_params, - query_params, + content, {"Accept": "*/*"}, response_type=object, auth_settings=["BearerToken"] @@ -124,13 +235,6 @@ def query_log_from_loki(k8s_client, context): assert response[0]['status'] == 'success' - result_data = response[0]['data']['result'] - - assert result_data, \ - 'No test log found in Loki with identifier={}'.format( - context['test_log_id'] - ) - assert result_data[0]['stream']['identifier'] == context['test_log_id'] - + return response # }}}