Avoid creating module-scoped app instance ; folder -> dir ; test refa…

…ctoring (#63)

.env.example

@@ -7,7 +7,7 @@ PORT=1416
 # Root path for the FastAPI app
 ROOT_PATH=""
 
-# Path to the folder containing the pipelines
+# Path to the directory containing the pipelines
 PIPELINES_DIR="pipelines"
 
 # Additional Python path to be added to the Python path

.gitignore

@@ -161,5 +161,5 @@ cython_debug/
 #.idea/
 
 
-# Pipelines default folder
+# Pipelines default directory
 /pipelines

src/hayhooks/cli/deploy_files/__init__.py

@@ -2,23 +2,23 @@
 import requests
 from pathlib import Path
 from urllib.parse import urljoin
-from hayhooks.server.utils.deploy_utils import read_pipeline_files_from_folder
+from hayhooks.server.utils.deploy_utils import read_pipeline_files_from_dir
 
 
 @click.command()
 @click.pass_obj
 @click.option('-n', '--name', required=True, help="Name of the pipeline to deploy")
-@click.argument('folder', type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path))
-def deploy_files(server_conf, name, folder):
-    """Deploy all pipeline files from a folder to the Hayhooks server."""
+@click.argument('pipeline_dir', type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path))
+def deploy_files(server_conf, name, pipeline_dir):
+    """Deploy pipeline files from a directory to the Hayhooks server."""
     server, disable_ssl = server_conf
 
     files_dict = {}
     try:
-        files_dict = read_pipeline_files_from_folder(folder)
+        files_dict = read_pipeline_files_from_dir(pipeline_dir)
 
         if not files_dict:
-            click.echo("Error: No valid files found in the specified folder")
+            click.echo("Error: No valid files found in the specified directory")
             return
 
         resp = requests.post(
@@ -31,4 +31,4 @@ def deploy_files(server_conf, name, folder):
             click.echo(f"Pipeline successfully deployed with name: {resp.json().get('name')}")
 
     except Exception as e:
-        click.echo(f"Error processing folder: {str(e)}")
+        click.echo(f"Error processing directory: {str(e)}")

src/hayhooks/cli/run/__init__.py

@@ -1,6 +1,7 @@
 import click
 import uvicorn
 import sys
+from hayhooks.server.app import create_app
 from hayhooks.settings import settings
 
 
@@ -20,4 +21,5 @@ def run(host, port, pipelines_dir, root_path, additional_python_path):
     if additional_python_path:
         sys.path.append(additional_python_path)
 
-    uvicorn.run("hayhooks.server:app", host=host, port=port)
+    app = create_app()
+    uvicorn.run(app, host=host, port=port)

src/hayhooks/server/__init__.py

@@ -1,4 +0,0 @@
-from hayhooks.server.app import app
-from hayhooks.server.routers import *
-
-__all__ = ["app"]

src/hayhooks/server/app.py

@@ -1,10 +1,12 @@
+from os import PathLike
+from typing import Union
 from fastapi import FastAPI
 from pathlib import Path
 from hayhooks.server.utils.deploy_utils import (
     deploy_pipeline_def,
     PipelineDefinition,
     deploy_pipeline_files,
-    read_pipeline_files_from_folder,
+    read_pipeline_files_from_dir,
 )
 from hayhooks.server.routers import status_router, draw_router, deploy_router, undeploy_router, openai_router
 from hayhooks.settings import settings
@@ -44,15 +46,40 @@ def deploy_files_pipeline(app: FastAPI, pipeline_dir: Path) -> dict:
         dict: Deployment result containing pipeline name
     """
     name = pipeline_dir.name
-    files = read_pipeline_files_from_folder(pipeline_dir)
+    files = read_pipeline_files_from_dir(pipeline_dir)
 
     if files:
         deployed_pipeline = deploy_pipeline_files(app, name, files)
-        log.info(f"Deployed pipeline from directory: {deployed_pipeline['name']}")
+        log.info(f"Deployed pipeline from dir: {deployed_pipeline['name']}")
         return deployed_pipeline
     return {"name": name}
 
 
+def create_pipeline_dir(pipelines_dir: Union[PathLike, str]):
+    """
+    Create a directory for pipelines if it doesn't exist.
+
+    If the directory doesn't exist, it will be created.
+    If the directory exists but is not a directory, an error will be raised.
+
+    Args:
+        pipelines_dir: Path to the pipelines directory
+
+    Returns:
+        str: Path to the pipelines directory
+    """
+    pipelines_dir = Path(pipelines_dir)
+
+    if not pipelines_dir.exists():
+        log.info(f"Creating pipelines dir: {pipelines_dir}")
+        pipelines_dir.mkdir(parents=True, exist_ok=True)
+
+    if not pipelines_dir.is_dir():
+        raise ValueError(f"pipelines_dir '{pipelines_dir}' exists but is not a directory")
+
+    return str(pipelines_dir)
+
+
 def create_app() -> FastAPI:
     """
     Create and configure a FastAPI application.
@@ -62,7 +89,7 @@ def create_app() -> FastAPI:
     - Includes all router endpoints (status, draw, deploy, undeploy)
     - Auto-deploys pipelines from the configured pipelines directory:
         - YAML pipeline definitions (*.yml, *.yaml)
-        - Pipeline folders containing multiple files
+        - Pipeline directories containing multiple files
 
     Returns:
         FastAPI: Configured FastAPI application instance
@@ -80,7 +107,7 @@ def create_app() -> FastAPI:
     app.include_router(openai_router)
 
     # Deploy all pipelines in the pipelines directory
-    pipelines_dir = settings.pipelines_dir
+    pipelines_dir = create_pipeline_dir(settings.pipelines_dir)
 
     if pipelines_dir:
         log.info(f"Pipelines dir set to: {pipelines_dir}")
@@ -99,26 +126,12 @@ def create_app() -> FastAPI:
                     continue
 
         if pipeline_dirs:
-            log.info(f"Deploying {len(pipeline_dirs)} pipeline(s) from folders")
+            log.info(f"Deploying {len(pipeline_dirs)} pipeline(s) from directories")
             for pipeline_dir in pipeline_dirs:
                 try:
                     deploy_files_pipeline(app, pipeline_dir)
                 except Exception as e:
-                    log.warning(f"Skipping pipeline folder {pipeline_dir}: {str(e)}")
+                    log.warning(f"Skipping pipeline directory {pipeline_dir}: {str(e)}")
                     continue
 
     return app
-
-
-app = create_app()
-
-
-@app.get("/")
-async def root():
-    return {
-        "swagger_docs": "http://localhost:1416/docs",
-        "deploy_pipeline": "http://localhost:1416/deploy",
-        "draw_pipeline": "http://localhost:1416/draw/{pipeline_name}",
-        "server_status": "http://localhost:1416/status",
-        "undeploy_pipeline": "http://localhost:1416/undeploy/{pipeline_name}",
-    }

src/hayhooks/server/utils/deploy_utils.py

@@ -63,15 +63,13 @@ async def pipeline_run(pipeline_run_req: PipelineRunRequest) -> JSONResponse:  #
     return {"name": pipeline_def.name}
 
 
-def save_pipeline_files(
-    pipeline_name: str, files: dict[str, str], pipelines_dir: str = settings.pipelines_dir
-) -> dict[str, str]:
+def save_pipeline_files(pipeline_name: str, files: dict[str, str], pipelines_dir: str) -> dict[str, str]:
     """Save pipeline files to disk and return their paths.
 
     Args:
         pipeline_name: Name of the pipeline
         files: Dictionary mapping filenames to their contents
-
+        pipelines_dir: Path to the pipelines directory
     Returns:
         Dictionary mapping filenames to their saved paths
 
@@ -81,6 +79,8 @@ def save_pipeline_files(
     try:
         # Create pipeline directory under the configured pipelines directory
         pipeline_dir = Path(pipelines_dir) / pipeline_name
+        log.debug(f"Creating pipeline dir: {pipeline_dir}")
+
         pipeline_dir.mkdir(parents=True, exist_ok=True)
         saved_files = {}
 
@@ -100,12 +100,12 @@ def save_pipeline_files(
         raise PipelineFilesError(f"Failed to save pipeline files: {str(e)}") from e
 
 
-def load_pipeline_module(pipeline_name: str, folder_path: Union[Path, str]) -> ModuleType:
-    """Load a pipeline module from a folder path.
+def load_pipeline_module(pipeline_name: str, dir_path: Union[Path, str]) -> ModuleType:
+    """Load a pipeline module from a directory path.
 
     Args:
         pipeline_name: Name of the pipeline
-        folder_path: Path to the folder containing the pipeline files
+        dir_path: Path to the directory containing the pipeline files
 
     Returns:
         The loaded module
@@ -114,8 +114,8 @@ def load_pipeline_module(pipeline_name: str, folder_path: Union[Path, str]) -> M
         ValueError: If the module cannot be loaded
     """
     try:
-        folder_path = Path(folder_path)
-        wrapper_path = folder_path / "pipeline_wrapper.py"
+        dir_path = Path(dir_path)
+        wrapper_path = dir_path / "pipeline_wrapper.py"
 
         if not wrapper_path.exists():
             raise PipelineWrapperError(f"Required file '{wrapper_path}' not found")
@@ -212,14 +212,14 @@ def deploy_pipeline_files(app: FastAPI, pipeline_name: str, files: dict[str, str
     if registry.get(pipeline_name):
         raise PipelineAlreadyExistsError(f"Pipeline '{pipeline_name}' already exists")
 
-    log.debug(f"Saving pipeline files for '{pipeline_name}'")
-    save_pipeline_files(pipeline_name, files=files)
+    log.debug(f"Saving pipeline files for '{pipeline_name}' in '{settings.pipelines_dir}'")
+    save_pipeline_files(pipeline_name, files=files, pipelines_dir=settings.pipelines_dir)
 
     pipeline_dir = Path(settings.pipelines_dir) / pipeline_name
     clog = log.bind(pipeline_name=pipeline_name, pipeline_dir=str(pipeline_dir), files=list(files.keys()))
 
     clog.debug("Loading pipeline module")
-    module = load_pipeline_module(pipeline_name, folder_path=pipeline_dir)
+    module = load_pipeline_module(pipeline_name, dir_path=pipeline_dir)
 
     clog.debug("Creating PipelineWrapper instance")
     pipeline_wrapper = create_pipeline_wrapper_instance(module)
@@ -288,27 +288,27 @@ def create_pipeline_wrapper_instance(pipeline_module: ModuleType) -> BasePipelin
     return pipeline_wrapper
 
 
-def read_pipeline_files_from_folder(folder_path: Path) -> dict[str, str]:
-    """Read pipeline files from a folder and return a dictionary mapping filenames to their contents.
+def read_pipeline_files_from_dir(dir_path: Path) -> dict[str, str]:
+    """Read pipeline files from a directory and return a dictionary mapping filenames to their contents.
     Skips directories, hidden files, and common Python artifacts.
 
     Args:
-        folder_path: Path to the folder containing the pipeline files
+        dir_path: Path to the directory containing the pipeline files
 
     Returns:
         Dictionary mapping filenames to their contents
     """
 
     files = {}
-    for file_path in folder_path.rglob("*"):
+    for file_path in dir_path.rglob("*"):
         if file_path.is_dir() or file_path.name.startswith('.'):
             continue
 
         if any(file_path.match(pattern) for pattern in settings.files_to_ignore_patterns):
             continue
 
         try:
-            files[str(file_path.relative_to(folder_path))] = file_path.read_text(encoding="utf-8", errors="ignore")
+            files[str(file_path.relative_to(dir_path))] = file_path.read_text(encoding="utf-8", errors="ignore")
         except Exception as e:
             log.warning(f"Skipping file {file_path}: {str(e)}")
             continue

src/hayhooks/settings.py

@@ -10,8 +10,9 @@ class AppSettings(BaseSettings):
     # Root path for the FastAPI app
     root_path: str = ""
 
-    # Path to the folder containing the pipelines
-    pipelines_dir: str = "pipelines"
+    # Path to the directory containing the pipelines
+    # Default to project root / pipelines
+    pipelines_dir: str = str(Path(__file__).parent.parent.parent / "pipelines")
 
     # Additional Python path to be added to the Python path
     additional_python_path: str = ""
@@ -22,20 +23,8 @@ class AppSettings(BaseSettings):
     # Port for the FastAPI app
     port: int = 1416
 
-    # Files to ignore when reading pipeline files from a folder
+    # Files to ignore when reading pipeline files from a directory
     files_to_ignore_patterns: list[str] = ["*.pyc", "*.pyo", "*.pyd", "__pycache__", "*.so", "*.egg", "*.egg-info"]
 
-    @field_validator("pipelines_dir")
-    def validate_pipelines_dir(cls, v):
-        path = Path(v)
-
-        if not path.exists():
-            path.mkdir(parents=True, exist_ok=True)
-
-        if not path.is_dir():
-            raise ValueError(f"pipelines_dir '{v}' exists but is not a directory")
-
-        return str(path)
-
 
 settings = AppSettings()

tests/conftest.py

@@ -1,12 +1,46 @@
 import pytest
+import shutil
+from pathlib import Path
+from fastapi import FastAPI
 from fastapi.testclient import TestClient
+from hayhooks.server.app import create_app
+from hayhooks.settings import settings
+from hayhooks.server.pipelines.registry import registry
+
+
+@pytest.fixture(scope="session", autouse=True)
+def test_settings():
+    settings.pipelines_dir = Path(__file__).parent / "pipelines"
+    return settings
+
+
+@pytest.fixture
+def test_app():
+    return create_app()
+
+
+@pytest.fixture
+def client(test_app: FastAPI):
+    return TestClient(test_app)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def cleanup_pipelines(test_settings):
+    """
+    This fixture is used to cleanup the pipelines directory
+    and the registry after each test module.
+    """
+    registry.clear()
+    if Path(test_settings.pipelines_dir).exists():
+        shutil.rmtree(test_settings.pipelines_dir)
 
 
 @pytest.fixture
 def deploy_pipeline():
     def _deploy_pipeline(client: TestClient, pipeline_name: str, pipeline_source_code: str):
         deploy_response = client.post("/deploy", json={"name": pipeline_name, "source_code": pipeline_source_code})
         return deploy_response
+
     return _deploy_pipeline
 
 
@@ -15,6 +49,7 @@ def undeploy_pipeline():
     def _undeploy_pipeline(client: TestClient, pipeline_name: str):
         undeploy_response = client.post(f"/undeploy/{pipeline_name}")
         return undeploy_response
+
     return _undeploy_pipeline
 
 
@@ -23,6 +58,7 @@ def draw_pipeline():
     def _draw_pipeline(client: TestClient, pipeline_name: str):
         draw_response = client.get(f"/draw/{pipeline_name}")
         return draw_response
+
     return _draw_pipeline
 
 
@@ -31,6 +67,7 @@ def status_pipeline():
     def _status_pipeline(client: TestClient, pipeline_name: str):
         status_response = client.get(f"/status/{pipeline_name}")
         return status_response
+
     return _status_pipeline
 
 
@@ -39,4 +76,5 @@ def deploy_files():
     def _deploy_files(client: TestClient, pipeline_name: str, pipeline_files: dict):
         deploy_response = client.post("/deploy_files", json={"name": pipeline_name, "files": pipeline_files})
         return deploy_response
+
     return _deploy_files