reduce cam memory usage

nlef · Nov 9, 2024 · ff174bc · ff174bc
1 parent 2f80a1c
commit ff174bc
Show file tree

Hide file tree

Showing 10 changed files with 72 additions and 58 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -91,7 +91,7 @@ jobs:
         name: Build and push mjpeg
         uses: docker/[email protected]
         with:
-          file: Dockerfile
+          file: Dockerfile-mjpeg
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta_mjpeg.outputs.tags }}
           labels: ${{ steps.meta_mjpeg.outputs.labels }}

diff --git a/Dockerfile b/Dockerfile
@@ -2,15 +2,13 @@ FROM python:3.12.7-slim-bookworm
 
 RUN apt update \
  && apt install -y \
-      python3-dev python3-setuptools python3-virtualenv \
-      python3-cryptography python3-opencv libuv1 \
-      ffmpeg x264 libx264-dev libwebp-dev \
-      libtiff5-dev libjpeg*-turbo libjpeg*-turbo-dev libopenjp2-7-dev zlib1g-dev \
-      libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \
-      libharfbuzz-dev libfribidi-dev libxcb1-dev \
+      ffmpeg x264 libx264-dev libuv1 \
+      libwebp-dev libtiff5-dev libjpeg*-turbo libjpeg*-turbo-dev libopenjp2-7-dev  \
+      zlib1g-dev libfreetype6-dev liblcms2-dev \
  && apt clean \
  && rm -rf /var/lib/apt/lists/*
 
+
 WORKDIR /opt
 
 RUN groupadd moonraker-telegram-bot --gid 1000 \
@@ -19,10 +17,10 @@ RUN groupadd moonraker-telegram-bot --gid 1000 \
  && chown -R moonraker-telegram-bot:moonraker-telegram-bot /opt/*
 
 COPY --chown=moonraker-telegram-bot:moonraker-telegram-bot . ./moonraker-telegram-bot
-RUN virtualenv -p /usr/bin/python3 --system-site-packages venv \
- && venv/bin/pip install --no-cache-dir -r moonraker-telegram-bot/scripts/requirements.txt
+RUN pip3 install --no-cache-dir -r moonraker-telegram-bot/scripts/requirements.docker.opencv.txt
+
 
 USER moonraker-telegram-bot
 VOLUME [ "/opt/printer_data/logs", "/opt/printer_data/config", "/opt/timelapse","/opt/timelapse_finished"]
-ENTRYPOINT ["/opt/venv/bin/python3", "moonraker-telegram-bot/bot/main.py"]
+ENTRYPOINT ["python3", "moonraker-telegram-bot/bot/main.py"]
 CMD ["-c", "/opt/printer_data/config/telegram.conf", "-l", "/opt/printer_data/logs"]
diff --git a/Dockerfile-dev b/Dockerfile-dev
@@ -0,0 +1,30 @@
+FROM python:3.12.7-bookworm
+
+RUN apt update \
+ && apt install -y \
+      ffmpeg x264 libx264-dev libuv1 \
+      libwebp-dev libtiff5-dev libjpeg*-turbo libjpeg*-turbo-dev libopenjp2-7-dev  \
+      zlib1g-dev libfreetype6-dev liblcms2-dev \
+      build-essential python3-dev libdebuginfod-dev libunwind-dev liblz4-dev \
+ && apt clean \
+ && rm -rf /var/lib/apt/lists/*
+
+
+WORKDIR /opt
+
+RUN groupadd moonraker-telegram-bot --gid 1000 \
+ && useradd moonraker-telegram-bot --uid 1000 --gid moonraker-telegram-bot \
+ && mkdir -p printer_data/logs printer_data/config timelapse timelapse_finished \
+ && chown -R moonraker-telegram-bot:moonraker-telegram-bot /opt/*
+
+COPY --chown=moonraker-telegram-bot:moonraker-telegram-bot . ./moonraker-telegram-bot
+RUN pip3 install --no-cache-dir -r moonraker-telegram-bot/scripts/requirements.docker.opencv.txt
+
+RUN pip3 install --no-cache-dir memray==1.14.0
+RUN pip3 install --no-cache-dir memory-profiler
+
+USER moonraker-telegram-bot
+VOLUME [ "/opt/printer_data/logs", "/opt/printer_data/config", "/opt/timelapse","/opt/timelapse_finished"]
+# ENTRYPOINT ["python3", "-m", "memray", "run", "--native", "--trace-python-allocators", "moonraker-telegram-bot/bot/main.py"]
+ENTRYPOINT ["python3", "moonraker-telegram-bot/bot/main.py"]
+CMD ["-c", "/opt/printer_data/config/telegram.conf", "-l", "/opt/printer_data/logs"]
diff --git a/Dockerfile_mjpeg → Dockerfile-mjpeg b/Dockerfile_mjpeg → Dockerfile-mjpeg
@@ -2,12 +2,9 @@ FROM python:3.12.7-slim-bookworm
 
 RUN apt update \
  && apt install -y \
-      python3-dev python3-setuptools python3-virtualenv \
-      python3-cryptography python3-numpy libuv1 \
-      ffmpeg x264 libx264-dev libwebp-dev \
-      libtiff5-dev libjpeg*-turbo libjpeg*-turbo-dev libopenjp2-7-dev zlib1g-dev \
-      libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \
-      libharfbuzz-dev libfribidi-dev libxcb1-dev \
+      ffmpeg x264 libx264-dev libuv1 \
+      libwebp-dev libtiff5-dev libjpeg*-turbo libjpeg*-turbo-dev libopenjp2-7-dev  \
+      zlib1g-dev libfreetype6-dev liblcms2-dev \
  && apt clean \
  && rm -rf /var/lib/apt/lists/*
 
@@ -19,10 +16,10 @@ RUN groupadd moonraker-telegram-bot --gid 1000 \
  && chown -R moonraker-telegram-bot:moonraker-telegram-bot /opt/*
 
 COPY --chown=moonraker-telegram-bot:moonraker-telegram-bot . ./moonraker-telegram-bot
-RUN virtualenv -p /usr/bin/python3 --system-site-packages venv \
- && venv/bin/pip install --no-cache-dir -r moonraker-telegram-bot/scripts/requirements.txt
+RUN pip3 install --no-cache-dir -r moonraker-telegram-bot/scripts/requirements.docker.mjpeg.txt
+
 
 USER moonraker-telegram-bot
 VOLUME [ "/opt/printer_data/logs", "/opt/printer_data/config", "/opt/timelapse","/opt/timelapse_finished"]
-ENTRYPOINT ["/opt/venv/bin/python3", "moonraker-telegram-bot/bot/main.py"]
+ENTRYPOINT ["python3", "moonraker-telegram-bot/bot/main.py"]
 CMD ["-c", "/opt/printer_data/config/telegram.conf", "-l", "/opt/printer_data/logs"]
diff --git a/bot/camera.py b/bot/camera.py
@@ -1,14 +1,14 @@
 import asyncio
 import functools
 from functools import wraps
-import gc
 import glob
 from io import BytesIO
 import logging
 import math
 import os
 import pathlib
 from pathlib import Path
+import pickle
 import threading
 import time
 from typing import List, Tuple
@@ -375,8 +375,7 @@ def process_video_frame(frame_local):
                 logger.debug("take_video cam read  frame execution time: %s millis", (time.time() - st_time) * 1000)
                 if time.time() > time_last_frame + frame_time:
                     time_last_frame = time.time()
-                    frame_list.append(frame_loc)
-                frame_loc = None
+                    frame_list.append(pickle.dumps(frame_loc))
                 del frame_loc
 
             self.cam_cam.release()
@@ -392,19 +391,16 @@ def process_video_frame(frame_local):
             )
 
             for el in frame_list:
-                out.write(process_video_frame(el))
-            for ii in range(len(frame_list)):  # pylint: disable=C0200
-                frame_list[ii] = None
+                loc_loc = pickle.loads(el)
+                out.write(process_video_frame(loc_loc))
+                del loc_loc
 
             out.release()
             del out
             os_nice(0)
 
-            del frame_list[:]
             frame_list.clear()
-            frame_list = None  # type: ignore
             del frame_list
-            gc.collect()
 
         video_bio = BytesIO()
         video_bio.name = "video.mp4"
@@ -722,8 +718,7 @@ def take_video(self) -> Tuple[BytesIO, BytesIO, int, int]:
                 logger.debug("take_video cam read  frame execution time: %s millis", (time.time() - st_time) * 1000)
                 if time.time() > time_last_frame + frame_time:
                     time_last_frame = time.time()
-                    frame_list.append(frame_loc)
-                frame_loc = None
+                    frame_list.append(pickle.dumps(frame_loc))
                 del frame_loc
 
             res_fps = len(frame_list) / self._video_duration
@@ -737,19 +732,16 @@ def take_video(self) -> Tuple[BytesIO, BytesIO, int, int]:
             )
 
             for el in frame_list:
-                out.write(self._image_to_frame(el))
-            for ii in range(len(frame_list)):  # pylint: disable=C0200
-                frame_list[ii] = None
+                loc_loc = pickle.loads(el)
+                out.write(self._image_to_frame(loc_loc))
+                del loc_loc
 
             out.release()
             del out
             os_nice(0)
 
-            del frame_list[:]
             frame_list.clear()
-            frame_list = None  # type: ignore
             del frame_list
-            gc.collect()
 
         video_bio = BytesIO()
         video_bio.name = "video.mp4"

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
@@ -0,0 +1,13 @@
+version: "3.9"
+services:
+  moonraker-telegram-bot:
+    platform: linux/arm64
+    build:
+        dockerfile:  Dockerfile
+        context: .
+    restart: unless-stopped
+    volumes:
+      - ./docker_data/timelapse:/opt/timelapse
+      - ./docker_data/timelapse_finished:/opt/timelapse_finished
+      - ./docker_data/log:/opt/printer_data/logs
+      - ./docker_data/config:/opt/printer_data/config
diff --git a/scripts/install.sh b/scripts/install.sh
@@ -121,29 +121,9 @@ stop_sevice() {
 
 install_packages() {
   PKGLIST=""
-
   report_status "Running apt-get update..."
   sudo apt-get update --allow-releaseinfo-change
-  INSTALL_OPENCV=""
-  while [[ ! (INSTALL_OPENCV =~ ^(?i)(y|n|no|yes)(?-i)$) ]]; do
-    read -p "Install python-opencv package? (Y/n): " -e -i "y" INSTALL_OPENCV
-    case "${INSTALL_OPENCV}" in
-      Y|y|Yes|yes)
-        echo -e "###### > Yes"
-        PKGLIST="python3-virtualenv python3-dev python3-cryptography python3-numpy python3-opencv libuv1 ffmpeg x264 libx264-dev libjpeg*-turbo libwebp-dev"
-        break;;
-      N|n|No|no)
-        echo -e "###### > No"
-        PKGLIST="python3-virtualenv python3-dev python3-cryptography python3-numpy libuv1 ffmpeg x264 libx264-dev libjpeg*-turbo libwebp-dev"
-        break;;
-      *)
-        warn_msg "Invalid command!";;
-    esac
-  done
-
-  for pkg in $PKGLIST; do
-    echo "${cyan}$pkg${default}"
-  done
+  PKGLIST="python3-virtualenv python3-numpy libuv1 ffmpeg x264 libx264-dev libjpeg*-turbo libwebp-dev"
   report_status "Installing packages..."
   sudo apt-get install --yes ${PKGLIST}
 }

diff --git a/scripts/requirements.docker.mjpeg.txt b/scripts/requirements.docker.mjpeg.txt
@@ -0,0 +1,2 @@
+-r requirements.txt
+numpy~=1.26.4
diff --git a/scripts/requirements.docker.opencv.txt b/scripts/requirements.docker.opencv.txt
@@ -0,0 +1,2 @@
+-r requirements.txt
+opencv-python~=4.10.0.84
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
@@ -6,7 +6,7 @@ orjson==3.10.11
 Pillow==11.0.0 ; python_version>='3.10'
 Pillow==10.4.0 ; python_version<='3.9'
 psutil==6.1.0
-python-telegram-bot[socks,http2,rate-limiter,callback-data,job-queue]==21.6
+python-telegram-bot[socks,http2,rate-limiter,callback-data,job-queue]==21.7
 tzlocal==2.1
 uvloop==0.20.0 ; platform_system != "Windows"
 websockets==13.1