Add service manager infrastructure

.cspell/frigate-dictionary.txt

@@ -42,6 +42,7 @@ codeproject
 colormap
 colorspace
 comms
+coro
 ctypeslib
 CUDA
 Cuvid
@@ -59,6 +60,7 @@ dsize
 dtype
 ECONNRESET
 edgetpu
+fastapi
 faststart
 fflags
 ffprobe
@@ -237,6 +239,7 @@ sleeptime
 SNDMORE
 socs
 sqliteq
+sqlitevecq
 ssdlite
 statm
 stimeout
@@ -271,6 +274,7 @@ unraid
 unreviewed
 userdata
 usermod
+uvicorn
 vaapi
 vainfo
 variations

frigate/app.py

@@ -63,6 +63,7 @@
 from frigate.record.export import migrate_exports
 from frigate.record.record import manage_recordings
 from frigate.review.review import manage_review_segments
+from frigate.service_manager import ServiceManager
 from frigate.stats.emitter import StatsEmitter
 from frigate.stats.util import stats_init
 from frigate.storage import StorageMaintainer
@@ -78,7 +79,6 @@
 
 class FrigateApp:
     def __init__(self, config: FrigateConfig) -> None:
-        self.audio_process: Optional[mp.Process] = None
         self.stop_event: MpEvent = mp.Event()
         self.detection_queue: Queue = mp.Queue()
         self.detectors: dict[str, ObjectDetectProcess] = {}
@@ -449,9 +449,8 @@ def start_audio_processor(self) -> None:
         ]
 
         if audio_cameras:
-            self.audio_process = AudioProcessor(audio_cameras, self.camera_metrics)
-            self.audio_process.start()
-            self.processes["audio_detector"] = self.audio_process.pid or 0
+            proc = AudioProcessor(audio_cameras, self.camera_metrics).start(wait=True)
+            self.processes["audio_detector"] = proc.pid or 0
 
     def start_timeline_processor(self) -> None:
         self.timeline_processor = TimelineProcessor(
@@ -639,11 +638,6 @@ def stop(self) -> None:
             ReviewSegment.end_time == None
         ).execute()
 
-        # stop the audio process
-        if self.audio_process:
-            self.audio_process.terminate()
-            self.audio_process.join()
-
         # ensure the capture processes are done
         for camera, metrics in self.camera_metrics.items():
             capture_process = metrics.capture_process
@@ -712,4 +706,6 @@ def stop(self) -> None:
             shm.close()
             shm.unlink()
 
+        ServiceManager.current().shutdown(wait=True)
+
         os._exit(os.EX_OK)

frigate/events/audio.py

@@ -9,7 +9,6 @@
 import numpy as np
 import requests
 
-import frigate.util as util
 from frigate.camera import CameraMetrics
 from frigate.comms.config_updater import ConfigSubscriber
 from frigate.comms.detections_updater import DetectionPublisher, DetectionTypeEnum
@@ -26,6 +25,7 @@
 from frigate.ffmpeg_presets import parse_preset_input
 from frigate.log import LogPipe
 from frigate.object_detection import load_labels
+from frigate.service_manager import ServiceProcess
 from frigate.util.builtin import get_ffmpeg_arg_list
 from frigate.video import start_or_restart_ffmpeg, stop_ffmpeg
 
@@ -63,13 +63,15 @@ def get_ffmpeg_command(ffmpeg: FfmpegConfig) -> list[str]:
     )
 
 
-class AudioProcessor(util.Process):
+class AudioProcessor(ServiceProcess):
+    name = "frigate.audio_manager"
+
     def __init__(
         self,
         cameras: list[CameraConfig],
         camera_metrics: dict[str, CameraMetrics],
     ):
-        super().__init__(name="frigate.audio_manager", daemon=True)
+        super().__init__()
 
         self.camera_metrics = camera_metrics
         self.cameras = cameras

frigate/mypy.ini

@@ -59,3 +59,7 @@ ignore_errors = false
 [mypy-frigate.watchdog]
 ignore_errors = false
 disallow_untyped_calls = false
+
+
+[mypy-frigate.service_manager.*]
+ignore_errors = false

frigate/service_manager/__init__.py

@@ -0,0 +1,4 @@
+from .multiprocessing import ServiceProcess
+from .service import Service, ServiceManager
+
+__all__ = ["Service", "ServiceProcess", "ServiceManager"]

frigate/service_manager/multiprocessing.py

@@ -0,0 +1,164 @@
+import asyncio
+import faulthandler
+import logging
+import multiprocessing as mp
+import signal
+import sys
+import threading
+from abc import ABC, abstractmethod
+from asyncio.exceptions import TimeoutError
+from logging.handlers import QueueHandler
+from types import FrameType
+from typing import Optional
+
+import frigate.log
+
+from .multiprocessing_waiter import wait as mp_wait
+from .service import Service, ServiceManager
+
+DEFAULT_STOP_TIMEOUT = 10  # seconds
+
+
+class BaseServiceProcess(Service, ABC):
+    """A Service the manages a multiprocessing.Process."""
+
+    _process: Optional[mp.Process]
+
+    def __init__(
+        self,
+        *,
+        name: Optional[str] = None,
+        manager: Optional[ServiceManager] = None,
+    ) -> None:
+        super().__init__(name=name, manager=manager)
+
+        self._process = None
+
+    async def on_start(self) -> None:
+        if self._process is not None:
+            if self._process.is_alive():
+                return  # Already started.
+            else:
+                self._process.close()
+
+        # At this point, the process is either stopped or dead, so we can recreate it.
+        self._process = mp.Process(target=self._run)
+        self._process.name = self.name
+        self._process.daemon = True
+        self.before_start()
+        self._process.start()
+        self.after_start()
+
+        self.manager.logger.info(f"Started {self.name} (pid: {self._process.pid})")
+
+    async def on_stop(
+        self,
+        *,
+        force: bool = False,
+        timeout: Optional[float] = None,
+    ) -> None:
+        if timeout is None:
+            timeout = DEFAULT_STOP_TIMEOUT
+
+        if self._process is None:
+            return  # Already stopped.
+
+        running = True
+
+        if not force:
+            self._process.terminate()
+            try:
+                await asyncio.wait_for(mp_wait(self._process), timeout)
+                running = False
+            except TimeoutError:
+                self.manager.logger.warning(
+                    f"{self.name} is still running after "
+                    f"{timeout} seconds. Killing."
+                )
+
+        if running:
+            self._process.kill()
+            await mp_wait(self._process)
+
+        self._process.close()
+        self._process = None
+
+        self.manager.logger.info(f"{self.name} stopped")
+
+    @property
+    def pid(self) -> Optional[int]:
+        return self._process.pid if self._process else None
+
+    def _run(self) -> None:
+        self.before_run()
+        self.run()
+        self.after_run()
+
+    def before_start(self) -> None:
+        pass
+
+    def after_start(self) -> None:
+        pass
+
+    def before_run(self) -> None:
+        pass
+
+    def after_run(self) -> None:
+        pass
+
+    @abstractmethod
+    def run(self) -> None:
+        pass
+
+    def __getstate__(self) -> dict:
+        return {
+            k: v
+            for k, v in self.__dict__.items()
+            if not (k.startswith("_Service__") or k == "_process")
+        }
+
+
+class ServiceProcess(BaseServiceProcess):
+    logger: logging.Logger
+
+    @property
+    def stop_event(self) -> threading.Event:
+        # Lazily create the stop_event. This allows the signal handler to tell if anyone is
+        # monitoring the stop event, and to raise a SystemExit if not.
+        if "stop_event" not in self.__dict__:
+            stop_event = threading.Event()
+            self.__dict__["stop_event"] = stop_event
+        else:
+            stop_event = self.__dict__["stop_event"]
+            assert isinstance(stop_event, threading.Event)
+
+        return stop_event
+
+    def before_start(self) -> None:
+        if frigate.log.log_listener is None:
+            raise RuntimeError("Logging has not yet been set up.")
+        self.__log_queue = frigate.log.log_listener.queue
+
+    def before_run(self) -> None:
+        super().before_run()
+
+        faulthandler.enable()
+
+        def receiveSignal(signalNumber: int, frame: Optional[FrameType]) -> None:
+            # Get the stop_event through the dict to bypass lazy initialization.
+            stop_event = self.__dict__.get("stop_event")
+            if stop_event is not None:
+                # Someone is monitoring stop_event. We should set it.
+                stop_event.set()
+            else:
+                # Nobody is monitoring stop_event. We should raise SystemExit.
+                sys.exit()
+
+        signal.signal(signal.SIGTERM, receiveSignal)
+        signal.signal(signal.SIGINT, receiveSignal)
+
+        self.logger = logging.getLogger(self.name)
+
+        logging.basicConfig(handlers=[], force=True)
+        logging.getLogger().addHandler(QueueHandler(self.__log_queue))
+        del self.__log_queue