Fix erroneous separation of camera frame production & consumption acr…

…oss processes

control/adafruithat/planktoscope/imagernew/__init__.py

@@ -48,10 +48,7 @@ def __init__(self, stop_event, exposure_time=10000, iso=100):
             )
             configuration = {}
 
-        self.__camera_type = "v2.1"
-
-        # parse the config data. If the key is absent, we are using the default value
-        self.__camera_type = configuration.get("camera_type", self.__camera_type)
+        self.__camera_type = configuration.get("camera_type", "v2.1")
 
         self.stop_event = stop_event
         self.__imager = planktoscope.imagernew.state_machine.Imager()
@@ -62,37 +59,12 @@ def __init__(self, stop_event, exposure_time=10000, iso=100):
         self.__pump_direction = "FORWARD"
         self.__img_goal = None
         self.imager_client = None
-        self.streaming_output = planktoscope.imagernew.picam_streamer.StreamingOutput()
         self.__error = 0
 
         # Initialize the camera
+        self.streaming_output = planktoscope.imagernew.picam_streamer.StreamingOutput()
         self.__camera = planktoscope.imagernew.picamera.picamera(self.streaming_output)
-
-        # Start the streaming
-        try:
-            self.__camera.start()
-        except Exception as e:
-            logger.exception(
-                f"An exception has occured when starting up picamera2: {e}"
-            )
-            try:
-                self.__camera.start(True)
-            except Exception as e:
-                logger.exception(
-                    f"A second exception has occured when starting up picamera2: {e}"
-                )
-                logger.error("This error can't be recovered from, terminating now")
-                raise e
-
-        """if self.__camera.sensor_name == "IMX219":  # Camera v2.1
-            self.__resolution = (3280, 2464)
-        elif self.__camera.sensor_name == "IMX477":  # Camera HQ
-            self.__resolution = (4056, 3040)
-        else:
-            self.__resolution = (1280, 1024)
-            logger.error(
-                f"The connected camera {self.__camera.sensor_name} is not recognized, please check your camera"
-            )"""
+        self.__resolution = None # this is set by the start method
 
         #self.__iso = iso
         self.__exposure_time = exposure_time
@@ -112,22 +84,6 @@ def __init__(self, stop_event, exposure_time=10000, iso=100):
         self.__export_path = ""
         self.__global_metadata = None
 
-        logger.info("Initialising the camera with the default settings")
-        # TODO identify the camera parameters that can be accessed and initialize them
-        self.__camera.exposure_time = self.__exposure_time
-        time.sleep(0.1)
-
-        self.__camera.exposure_mode = self.__exposure_mode
-        time.sleep(0.1)
-
-        self.__camera.white_balance = self.__white_balance
-        time.sleep(0.1)
-
-        self.__camera.white_balance_gain = self.__white_balance_gain
-        time.sleep(0.1)
-
-        self.__camera.image_gain = self.__image_gain
-
         logger.success("planktoscope.imager is initialised and ready to go!")
 
     # copied #
@@ -239,7 +195,48 @@ def run(self):
                 "status/imager", '{"camera_name":"Not recognized"}'
             )
 
-        logger.info("Starting the streaming server thread")
+        # Note(ethanjli): the camera must be started in the same process as anything which uses
+        # self.streaming_output, such as our StreamingHandler. This is because
+        # self.streaming_output does not synchronize state across independent processes!
+        # TODO(ethanjli): it would be cleaner if we can start the camera and the StreamingServer
+        # separately from the MQTT client; if it's possible, we can figure that out later.
+        try:
+            self.__camera.start()
+        except Exception as e:
+            logger.exception(
+                f"An exception has occured when starting up picamera2: {e}"
+            )
+            try:
+                self.__camera.start(True)
+            except Exception as e:
+                logger.exception(
+                    f"A second exception has occured when starting up picamera2: {e}"
+                )
+                logger.error("This error can't be recovered from, terminating now")
+                raise e
+
+        logger.info("Initialising the camera with the default settings...")
+        # TODO identify the camera parameters that can be accessed and initialize them
+        self.__camera.exposure_time = self.__exposure_time
+        time.sleep(0.1)
+        self.__camera.exposure_mode = self.__exposure_mode
+        time.sleep(0.1)
+        self.__camera.white_balance = self.__white_balance
+        time.sleep(0.1)
+        self.__camera.white_balance_gain = self.__white_balance_gain
+        time.sleep(0.1)
+        self.__camera.image_gain = self.__image_gain
+
+        """if self.__camera.sensor_name == "IMX219":  # Camera v2.1
+            self.__resolution = (3280, 2464)
+        elif self.__camera.sensor_name == "IMX477":  # Camera HQ
+            self.__resolution = (4056, 3040)
+        else:
+            self.__resolution = (1280, 1024)
+            logger.error(
+                f"The connected camera {self.__camera.sensor_name} is not recognized, please check your camera"
+            )"""
+
         try:
             address = ("", 8000)
             fps = 15

control/adafruithat/planktoscope/imagernew/picamera.py

@@ -25,13 +25,20 @@
 ################################################################################
 class picamera:
     def __init__(self, output, *args, **kwargs):
-        self.__picam = Picamera2()
+        # Note(ethanjli): if we instantiate Picamera2 here in one process and then call the start
+        # method from a child process, then the start method's call of self.__picam.configure
+        # will block forever because we've basically duplicated the Picamera2 object when we forked
+        # into a child process. So instead we initialize self.__picam to None here, and we'll
+        # properly initialize self.__picam in the start method, which is called by a different
+        # process than the process which calls __init__.
+        self.__picam = None
         self.__controls = {}
         self.__output = output
         self.__sensor_name = ""
 
     # TODO decide which stream to display (main, lores or raw)
     def start(self, force=False):
+        self.__picam = Picamera2()
         logger.debug("Starting up picamera2")
         if force:
             # let's close the camera first
@@ -44,6 +51,12 @@ def start(self, force=False):
         half_resolution = [dim // 2 for dim in self.__picam.sensor_resolution]
         main_stream = {"size": half_resolution}
         lores_stream = {"size": (640, 480)}
+        # Note(ethanjli): if we use "lores" as our encode argument, we must use the MJPEGEncoder
+        # instead of JpegEncoder. This is because on the RPi4 the lores stream only outputs as
+        # YUV420, but JpegEncoder only accepts RGB. By contrast, MJPEGEncoder can handle YUV420.
+        # If we do need RGB output for something, we'll have to use the "main" stream instead of the
+        # "lores" stream for that. For details, refer to Table 1 on page 59 of the picamera2 manual
+        # at https://datasheets.raspberrypi.com/camera/picamera2-manual.pdf.
         config = self.__picam.create_video_configuration(main_stream, lores_stream, encode="lores")
         self.__picam.configure(config)
 
@@ -53,8 +66,10 @@ def start(self, force=False):
         self.__height = self.__picam.sensor_resolution[1]
 
         # Start recording with video encoding and writing video frames
-        self.__picam.start_preview(Preview.QT)
-        self.__picam.start_recording(JpegEncoder(), FileOutput(self.__output), Quality.HIGH)
+        # Note(ethanjli): see note above about JpegEncoder vs. MJPEGEncoder compatibility with
+        # "lores" streams!
+        #self.__picam.start_recording(JpegEncoder(), FileOutput(self.__output), Quality.HIGH)
+        self.__picam.start_recording(MJPEGEncoder(), FileOutput(self.__output), Quality.HIGH)
 
     @property
     def sensor_name(self):

control/pyproject.toml

@@ -58,7 +58,7 @@ loguru = "~0.5.3"
 picamerax = "~21.9.8"
 smbus2 = "~0.4.1"
 picamera = "~1.13"
-picamera2 = "~0.3.17"
+picamera2 = "==0.3.10"
 # Note: av is required by picamera2; v10.0.0 is the latest version available on piwheels for bullseye, while v11.0.0 is only for bookworm
 av = "~10.0.0"
 pillow = "~10.2.0"