From 505d727c7605aa155276d90b17c2c3b964f23584 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 11 Sep 2024 21:32:01 +0700 Subject: [PATCH] fix: running test on windows Signed-off-by: James --- engine/e2e-test/test_api_engine_list.py | 4 +- engine/e2e-test/test_cli_engine_uninstall.py | 2 +- engine/e2e-test/test_runner.py | 121 ++++++++++++++++--- 3 files changed, 105 insertions(+), 22 deletions(-) diff --git a/engine/e2e-test/test_api_engine_list.py b/engine/e2e-test/test_api_engine_list.py index b09919cf3..974fbbf8e 100644 --- a/engine/e2e-test/test_api_engine_list.py +++ b/engine/e2e-test/test_api_engine_list.py @@ -8,7 +8,9 @@ class TestApiEngineList: @pytest.fixture(autouse=True) def setup_and_teardown(self): # Setup - start_server() + success = start_server() + if not success: + raise Exception("Failed to start server") yield diff --git a/engine/e2e-test/test_cli_engine_uninstall.py b/engine/e2e-test/test_cli_engine_uninstall.py index 7b6a1f488..03078a1e6 100644 --- a/engine/e2e-test/test_cli_engine_uninstall.py +++ b/engine/e2e-test/test_cli_engine_uninstall.py @@ -14,7 +14,7 @@ def setup_and_teardown(self): # Teardown # Clean up, removing installed engine - run("Uninstall Engine", ["engines", "uninsatll", "cortex.llamacpp"]) + run("Uninstall Engine", ["engines", "uninstall", "cortex.llamacpp"]) def test_engines_uninstall_llamacpp_should_be_successfully(self): exit_code, output, error = run( diff --git a/engine/e2e-test/test_runner.py b/engine/e2e-test/test_runner.py index c1b0a1d83..4d3390b54 100644 --- a/engine/e2e-test/test_runner.py +++ b/engine/e2e-test/test_runner.py @@ -1,36 +1,55 @@ import platform +import queue import select import subprocess +import threading import time from typing import List +# You might want to change the path of the executable based on your build directory +executable_windows_path = "build\\Debug\\cortex.exe" +executable_unix_path = "build/cortex" -def run(name: str, arguments: List[str]): +# Timeout +timeout = 5 # secs +start_server_success_message = "Server started" + + +# Get the executable path based on the platform +def getExecutablePath() -> str: if platform.system() == "Windows": - executable = "build\\cortex-cpp.exe" + return executable_windows_path else: - executable = "build/cortex-cpp" - print("Command name", name) - print("Running command: ", [executable] + arguments) - if len(arguments) == 0: - result = subprocess.run(executable, capture_output=True, text=True, timeout=5) - else: - result = subprocess.run( - [executable] + arguments, capture_output=True, text=True, timeout=5 - ) + return executable_unix_path + + +# Execute a command +def run(test_name: str, arguments: List[str]): + executable_path = getExecutablePath() + print("Running:", test_name) + print("Command:", [executable_path] + arguments) + + result = subprocess.run( + [executable_path] + arguments, capture_output=True, text=True, timeout=timeout + ) return result.returncode, result.stdout, result.stderr -def start_server(timeout=5): +# Start the API server +# Wait for `Server started` message or failed +def start_server() -> bool: if platform.system() == "Windows": - executable = "build\\cortex-cpp.exe" + return start_server_windows() else: - executable = "build/cortex-cpp" + return start_server_nix() + + +def start_server_nix() -> bool: + executable = getExecutablePath() process = subprocess.Popen( executable, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) - success_message = "Server started" start_time = time.time() while time.time() - start_time < timeout: # Use select to check if there's data to read from stdout or stderr @@ -39,18 +58,80 @@ def start_server(timeout=5): for stream in readable: line = stream.readline() if line: - print(line.strip()) # Print output for debugging - if success_message in line: + if start_server_success_message in line: # have to wait a bit for server to really up and accept connection + print("Server started found, wait 0.3 sec..") time.sleep(0.3) - return True, process # Success condition met + return True # Check if the process has ended if process.poll() is not None: - return False, process # Process ended without success message + return False + + return False + + +def start_server_windows() -> bool: + executable = getExecutablePath() + process = subprocess.Popen( + executable, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + universal_newlines=True, + ) + + q_out = queue.Queue() + q_err = queue.Queue() + + def enqueue_output(out, queue): + for line in iter(out.readline, b""): + queue.put(line) + out.close() + + # Start threads to read stdout and stderr + t_out = threading.Thread(target=enqueue_output, args=(process.stdout, q_out)) + t_err = threading.Thread(target=enqueue_output, args=(process.stderr, q_err)) + t_out.daemon = True + t_err.daemon = True + t_out.start() + t_err.start() + + # only wait for defined timeout + start_time = time.time() + while time.time() - start_time < timeout: + # Check stdout + try: + line = q_out.get_nowait() + except queue.Empty: + pass + else: + print(f"STDOUT: {line.strip()}") + if start_server_success_message in line: + return True + + # Check stderr + try: + line = q_err.get_nowait() + except queue.Empty: + pass + else: + print(f"STDERR: {line.strip()}") + if start_server_success_message in line: + # found the message. let's wait for some time for the server successfully started + time.sleep(0.3) + return True, process + + # Check if the process has ended + if process.poll() is not None: + return False + + time.sleep(0.1) - return False, process # Timeout reached + return False +# Stop the API server def stop_server(): run("Stop server", ["stop"])