Add opensearch cluster stdout and stderr logs to test results manifest (

#4352) Signed-off-by: Rishabh Singh <[email protected]>
opensearch-project · Jan 25, 2024 · a7f0228 · a7f0228
1 parent 0fe9758
commit a7f0228
Show file tree

Hide file tree

Showing 6 changed files with 323 additions and 26 deletions.
diff --git a/src/manifests/test_report_manifest.py b/src/manifests/test_report_manifest.py
@@ -30,6 +30,10 @@ class TestReportManifest(ComponentManifest['TestReportManifest', 'TestComponents
               - name: with-security
                 status: the status of the test run with this config. e.g. pass/fail
                 yml: URL or local path to the component yml file
+                cluster_stdout:
+                  - URL or local path to the OpenSearch cluster logs
+                cluster_stderr:
+                  - URL or local path to the OpenSearch cluster error logs
     """
 
     SCHEMA = {
@@ -61,6 +65,8 @@ class TestReportManifest(ComponentManifest['TestReportManifest', 'TestComponents
                                 "name": {"type": "string"},
                                 "status": {"type": "string"},
                                 "yml": {"type": "string"},
+                                "cluster_stdout": {"type": "list"},
+                                "cluster_stderr": {"type": "list"}
                             }
                         },
                     },
@@ -140,12 +146,16 @@ def __init__(self, data: dict) -> None:
                 self.name = data["name"]
                 self.status = data["status"]
                 self.yml = data["yml"]
+                self.cluster_stdout = data["cluster_stdout"]
+                self.cluster_stderr = data["cluster_stderr"]
 
             def __to_dict__(self) -> dict:
                 return {
                     "name": self.name,
                     "status": self.status,
-                    "yml": self.yml
+                    "yml": self.yml,
+                    "cluster_stdout": self.cluster_stdout,
+                    "cluster_stderr": self.cluster_stderr
                 }
 
 

diff --git a/src/report_workflow/test_report_runner.py b/src/report_workflow/test_report_runner.py
@@ -7,6 +7,7 @@
 
 import logging
 import os
+import typing
 import urllib.request
 from typing import Any
 from urllib.error import HTTPError
@@ -100,6 +101,8 @@ def component_entry(self, component_name: str) -> Any:
                 component_yml_ref = "URL not available"
             config_dict["yml"] = component_yml_ref
             config_dict["status"] = test_result
+            config_dict["cluster_stdout"] = get_os_cluster_logs(self.base_path, str(self.test_run_id), self.test_type, component_name, config, self.name)[0]
+            config_dict["cluster_stderr"] = get_os_cluster_logs(self.base_path, str(self.test_run_id), self.test_type, component_name, config, self.name)[1]
             component["configs"].append(config_dict)
         return component
 
@@ -120,11 +123,14 @@ def test_report_manifest_data_template(self, template_type: str) -> Any:
         return templates[template_type]
 
 
-def generate_component_yml_ref(base_path: str, test_number: str, test_type: str, component_name: str, config: str) -> str:
+def generate_component_yml_ref(base_path: str, test_number: str, test_type: str, component_name: str,
+                               config: str) -> str:
     if base_path.startswith("https://"):
-        return "/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, f"{component_name}.yml"])
+        return "/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config,
+                         f"{component_name}.yml"])
     else:
-        return os.path.join(base_path, "test-results", test_number, test_type, component_name, config, f"{component_name}.yml")
+        return os.path.join(base_path, "test-results", test_number, test_type, component_name, config,
+                            f"{component_name}.yml")
 
 
 def generate_test_command(test_type: str, test_manifest_path: str, artifacts_path: str, component: str = "") -> str:
@@ -135,4 +141,26 @@ def generate_test_command(test_type: str, test_manifest_path: str, artifacts_pat
     return command
 
 
+def get_os_cluster_logs(base_path: str, test_number: str, test_type: str, component_name: str, config: str,
+                        product_name: str) -> typing.List[list]:
+    os_stdout: list = []
+    os_stderr: list = []
+    cluster_ids: list
+
+    if product_name == 'opensearch':
+        cluster_ids = ['id-0'] if config == 'with-security' else ['id-1']
+    else:
+        cluster_ids = ['id-0', 'id-1'] if config == 'with-security' else ['id-2', 'id-3']
+
+    for ids in cluster_ids:
+        if base_path.startswith("https://"):
+            os_stdout.append("/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, f"local-cluster-logs/{ids}/stdout.txt"]))
+            os_stderr.append("/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, f"local-cluster-logs/{ids}/stderr.txt"]))
+        else:
+            os_stdout.append(os.path.join(base_path, "test-results", test_number, test_type, component_name, config, f"local-cluster-logs/{ids}/stdout.txt"))
+            os_stderr.append(os.path.join(base_path, "test-results", test_number, test_type, component_name, config, f"local-cluster-logs/{ids}/stderr.txt"))
+
+    return [os_stdout, os_stderr]
+
+
 TestReportRunner.__test__ = False  # type:ignore