#414: Ignore rsyslogd related errors in db (#415)

fixes #414

doc/changes/changes_3.2.0.md

@@ -1,8 +1,8 @@
-# Integration-Test-Docker-Environment 3.2.0, released t.b.d.
+# Integration-Test-Docker-Environment 3.2.0, released 2024-09-18
 
 ## Summary
 
-Updated dependency constraints and supported Exasol versions.
+Updated dependency constraints and supported Exasol versions. Also, ignore crashes of rsyslogd in the docker-db.
 
 ### Supported Exasol Versions
 
@@ -12,4 +12,6 @@ Updated dependency constraints and supported Exasol versions.
 ## Dependencies
 
 ## Changes
+
 #412: Add latest Docker-DB versions 
+#414: Ignore rsyslogd related errors in db

exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py

@@ -2,6 +2,7 @@
 import time
 from pathlib import Path
 from threading import Thread
+from typing import Callable, Optional
 
 from docker.models.containers import Container
 
@@ -21,6 +22,28 @@ def __init__(self, container: Container, logger, log_file: Path, description: st
         self.previous_timestamp = None
         self.current_timestamp = None
         self.error_message = None
+        self.ignore_error_return_codes = (
+            "(membership) returned with state 1",  # exclude webui not found in 7.0.0
+            "rsyslogd) returned with state 1"  # exclude rsyslogd which might crash when running itde under lima
+        )
+
+    def _contains_error(self, log_line: str) -> bool:
+        def ignore_sshd(log_line_local):
+            return "sshd was not started" in log_line_local
+
+        def ignore_return_code(log_line_local):
+            return any(x in log_line_local for x in self.ignore_error_return_codes)
+
+        def contains(substr: str, ignore: Optional[Callable[[str], bool]] = None):
+            if not substr in log_line:
+                return False
+            return ignore is None or not ignore(log_line)
+
+        return (
+                contains("error", ignore_sshd)
+                or contains("exception")
+                or contains("returned with state 1", ignore_return_code)
+        )
 
     def stop(self):
         self.logger.info("Stop ContainerLogThread")
@@ -38,10 +61,7 @@ def run(self):
                         still_running_logger.log()
                         log_handler.handle_log_lines(log)
                     log_line = log.decode("utf-8").lower()
-                    if ("error" in log_line and not "sshd was not started" in log_line) \
-                            or "exception" in log_line \
-                            or ("returned with state 1" in log_line
-                                    and not "(membership) returned with state 1" in log_line): # exclude webui not found in 7.0.0
+                    if self._contains_error(log_line):
                         self.logger.info("ContainerLogHandler error message, %s", log_line)
                         self.error_message = log_line
                         self.finish = True
@@ -50,4 +70,4 @@ def run(self):
                     time.sleep(1)
         except Exception as e:
             self.finish = True
-            self.logger.exception("Caught exception in DBContainerLogThread.run.")
+            self.logger.exception("Caught exception in DBContainerLogThread.run.")

noxfile.py

@@ -201,7 +201,8 @@ def run_minimal_tests(session: nox.Session, db_version: str):
             "test_termination_handler.py",
         ],
         "new-itest": [
-            "test_cli_environment.py"
+            "test_cli_environment.py",
+            "test_db_container_log_thread.py"
             ],
         "unit": "./test/unit",
     }

test/integration/test_db_container_log_thread.py

@@ -0,0 +1,99 @@
+import logging
+import tempfile
+import time
+from pathlib import Path
+from typing import List
+
+import pytest
+
+from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient
+from exasol_integration_test_docker_environment.lib.test_environment.database_waiters.db_container_log_thread import \
+    DBContainerLogThread
+
+def _build_docker_command(logs: List[str]):
+    """
+    Builds a bash while loop command which can be used to print logs.
+    Args:
+        logs: List of logs to print, each in one line.
+
+    Returns:
+        Something like ["bash", "-c", "while true; do echo 'Test'; done"]
+    """
+    echo_commands = [f"echo '{log}'" for log in logs]
+    echo_commdands_str = ";".join(echo_commands)
+    bash_command = f"while true; do {echo_commdands_str}; sleep 0.1; done"
+    return ["bash", "-c", bash_command]
+
+def _run_container_log_thread(logger, logs: List[str]) -> str:
+    """
+    Starts a dummy docker container which prints logs in an endless loop, and calls DBContainerLogThread on that container.
+
+    Returns: resulting DBContainerLogThread.error_message
+    """
+    with tempfile.TemporaryDirectory() as tmpDir:
+        with ContextDockerClient(timeout=3600) as client:
+            try:
+                container = client.containers.run("ubuntu", _build_docker_command(logs), detach=True)
+                thread = DBContainerLogThread(container, logger, Path(tmpDir) / "log.txt", "test")
+                thread.start()
+                time.sleep(2)
+                thread.stop()
+            finally:
+                container.stop()
+                container.remove()
+            return thread.error_message
+
+
+@pytest.fixture
+def test_logger():
+    return logging.Logger(__name__)
+
+def test_container_log_thread_no_error(test_logger) -> None:
+    """
+    Integration test which verifies that the DBContainerLogThread returns no error message if no error is logged.
+    """
+    error_message = _run_container_log_thread(test_logger, ["test", "something", "db started"])
+    assert error_message is None
+
+def test_container_log_thread_error(test_logger) -> None:
+    """
+    Integration test which verifies that the DBContainerLogThread returns error message if error is logged.
+    """
+    error_message = _run_container_log_thread(test_logger, ["confd returned with state 1"])
+    assert "confd returned with state 1\n" in error_message
+
+def test_container_log_thread_ignore_rsyslogd(test_logger) -> None:
+    """
+    Integration test which verifies that the DBContainerLogThread returns no error message if rsyslogd crashes.
+    """
+    rsys_logd_logs = [
+        "[2024-09-17 14:12:20.335085 +00:00] child 58687 (Part:9 Node:0 rsyslogd) returned with state 1.",
+        "[2024-09-17 14:12:20.336886 +00:00] Started /sbin/rsyslogd with PID:58688 UID:0 GID:0 Part:9 Node:0",
+        "[2024-09-17 14:12:20.336967 +00:00] 30 auto-restarted processes exited in the last 0 seconds. Starting to delay process death handling."
+    ]
+    error_message = _run_container_log_thread(test_logger, rsys_logd_logs)
+    assert error_message is None
+
+def test_container_log_thread_ignore_sshd(test_logger) -> None:
+    """
+    Integration test which verifies that the DBContainerLogThread returns no error message if sshd crashes.
+    """
+    sshd_logs = [
+        "[2024-09-17 14:12:20.335085 +00:00] error : sshd was not started.",
+    ]
+    error_message = _run_container_log_thread(test_logger, sshd_logs)
+    assert error_message is None
+
+def test_container_log_thread_exception(test_logger) -> None:
+    """
+    Integration test which verifies that the DBContainerLogThread returns an error message if an exception was thrown.
+    """
+    sshd_logs = [
+        "Traceback (most recent call last):",
+        'File "/opt/cos/something.py", line 364, in runcode',
+        '    coro = func()',
+        '  File "<input>", line 1, in <module>',
+        'Exception: bad thing happend'
+    ]
+    error_message = _run_container_log_thread(test_logger, sshd_logs)
+    assert "exception: bad thing happend\n" in error_message