tests: add scenario for logging pipeline

spawn a container that logs a single line
then try to retrieve it from Loki API

Refs: #2701

tests/post/features/logging.feature

@@ -7,9 +7,16 @@ Feature: Logging stack is up and running
   Scenario: Expected Pods
     Given the Kubernetes API is available
     Then we have 1 running pod labeled 'app=loki' in namespace 'metalk8s-logging'
+    And we have 1 running pod labeled 'app=fluent-bit' in namespace 'metalk8s-logging' on node 'bootstrap'
 
   Scenario: Pushing log to Loki directly
     Given the Kubernetes API is available
     And the Loki API is available
     When we push an example log to Loki
     Then we can query this example log from Loki
+
+  Scenario: Logging pipeline is working
+    Given the Kubernetes API is available
+    And the Loki API is available
+    And we have set up a 'logger-test' pod
+    Then we can retrieve logs from 'logger-test' pod in Loki API

tests/post/steps/files/logger-pod.yaml.tpl

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: $name
+  namespace: metalk8s-logging
+spec:
+  tolerations:
+  - key: "node-role.kubernetes.io/bootstrap"
+    operator: "Equal"
+    effect: "NoSchedule"
+  - key: "node-role.kubernetes.io/infra"
+    operator: "Equal"
+    effect: "NoSchedule"
+  containers:
+    - name: $name
+      image: $image
+      command:
+      - echo
+      - logging pipeline is working
+  restartPolicy: Never

tests/post/steps/test_logging.py

@@ -1,14 +1,26 @@
-import json
+import pathlib
+import string
 import time
 import uuid
 
 import pytest
-from pytest_bdd import scenario, given, when, then
+from pytest_bdd import scenario, given, when, then, parsers
 
-from tests import utils
+from tests import utils, kube_utils
+
+# Constants {{{
+
+MANIFESTS_PATH = pathlib.Path("/etc/kubernetes/manifests/")
+TEMPLATES_PATH = pathlib.Path("/srv/scality/_templates/")
+LOGGER_POD_TEMPLATE = (
+    pathlib.Path(__file__) / ".." / "files" / "logger-pod.yaml.tpl"
+).resolve()
+
+# }}}
 
 # Fixtures {{{
 
+
 @pytest.fixture(scope='function')
 def context():
     return {}
@@ -33,6 +45,11 @@ def test_push_log_to_loki(host):
     pass
 
 
+@scenario('../features/logging.feature', 'Logging pipeline is working')
+def test_logging_pipeline_is_working(host):
+    pass
+
+
 # }}}
 # Given {{{
 
@@ -56,6 +73,60 @@ def _check_loki_ready():
     )
 
 
+@given(parsers.parse("we have set up a '{pod_name}' pod"),
+       target_fixture='pod_creation_ts')
+def set_up_logger_pod(host, nodename, k8s_client, utils_image, pod_name):
+    manifest_template = LOGGER_POD_TEMPLATE.read_text(encoding="utf-8")
+    manifest = string.Template(manifest_template).substitute(
+        name=pod_name,
+        image=utils_image,
+    )
+    manifest_path = str(MANIFESTS_PATH / "{}.yaml".format(pod_name))
+    template_path = str(
+        TEMPLATES_PATH / "{}.yaml".format(pod_name)
+    )
+
+    with host.sudo():
+        host.run_test("mkdir -p %s", str(TEMPLATES_PATH))
+
+        write_template = utils.write_string(host, template_path, manifest)
+        assert write_template.rc == 0, (
+            "Failed to create logger Pod manifest template '{}': {}"
+        ).format(template_path, write_template.stderr)
+
+        pod_creation_ts = int(time.time())
+        manage_static_pod = host.run(
+            "salt-call state.single metalk8s.static_pod_managed "
+            'name=%s source=%s',
+            manifest_path,
+            template_path,
+        )
+
+        assert manage_static_pod.rc == 0, (
+            "Failed to manage logger Pod with Salt: {}"
+        ).format(manage_static_pod.stderr)
+
+    pod_fullname = "{}-{}".format(pod_name, nodename)
+    utils.retry(
+        kube_utils.check_pod_status(
+            k8s_client, pod_fullname,
+            namespace="metalk8s-logging", state="Succeeded",
+        ),
+        times=10,
+        wait=5,
+        name="wait for Pod '{}'".format(pod_fullname),
+    )
+
+    yield pod_creation_ts
+
+    with host.sudo():
+        for filename in (manifest_path, template_path):
+            cmd = host.run("rm -f %s", filename)
+            assert cmd.rc == 0, "Failed to remove file '{}': {}".format(
+                filename, cmd.stderr
+            )
+
+
 # }}}
 # When {{{
 
@@ -102,35 +173,68 @@ def push_log_to_loki(k8s_client, context):
 
 @then("we can query this example log from Loki")
 def query_log_from_loki(k8s_client, context):
+    query = {'query': '{{identifier="{0}"}}'.format(context['test_log_id'])}
+    response = query_loki_api(k8s_client, query)
+    result_data = response[0]['data']['result']
+
+    assert result_data, \
+        'No test log found in Loki with identifier={}'.format(
+            context['test_log_id']
+        )
+    assert result_data[0]['stream']['identifier'] == context['test_log_id']
+
+
+@then(parsers.parse("we can retrieve logs from '{pod_name}' pod in Loki API"))
+def retrieve_pod_logs_from_loki(k8s_client, nodename, pod_name,
+                                pod_creation_ts):
+    pod_fullname = "{}-{}".format(pod_name, nodename)
+    query = {
+        'query': '{{instance="{0}"}}'.format(pod_fullname),
+        'start': pod_creation_ts,
+    }
+
+    def _check_log_line_exists():
+        response = query_loki_api(k8s_client, query, route='query_range')
+        try:
+            result_data = response[0]['data']['result'][0]['values']
+        except IndexError:
+            result_data = []
+        assert any("logging pipeline is working" in v[1]
+                   for v in result_data), \
+            'No log found in Loki for pod "{}"'.format(pod_fullname)
+
+    utils.retry(
+        _check_log_line_exists,
+        times=20,
+        wait=3,
+        name="check that a log exists for pod '{0}'".format(pod_fullname)
+    )
+
+# }}}
+
+# Helpers {{{
+
+
+def query_loki_api(k8s_client, content, route='query'):
     # With current k8s client we cannot pass query_params so we need to
     # use `call_api` directly
     path_params = {
         'name': 'loki:http-metrics',
         'namespace': 'metalk8s-logging',
-        'path': 'loki/api/v1/query'
-    }
-    query_params = {
-        'query': '{identifier="' + context['test_log_id'] + '"}'
+        'path': 'loki/api/v1/{0}'.format(route)
     }
     response = k8s_client.api_client.call_api(
         '/api/v1/namespaces/{namespace}/services/{name}/proxy/{path}',
         'GET',
         path_params,
-        query_params,
+        content,
         {"Accept": "*/*"},
         response_type=object,
         auth_settings=["BearerToken"]
     )
 
     assert response[0]['status'] == 'success'
 
-    result_data = response[0]['data']['result']
-
-    assert result_data, \
-        'No test log found in Loki with identifier={}'.format(
-            context['test_log_id']
-        )
-    assert result_data[0]['stream']['identifier'] == context['test_log_id']
-
+    return response
 
 # }}}