feat: add rq worker

.deepsource.toml

@@ -12,8 +12,8 @@ runtime_version = "3.x.x"
 
 [[transformers]]
 name = "black"
-enabled = true
+enabled = false
 
 [[transformers]]
 name = "isort"
-enabled = true
+enabled = false

.env

@@ -46,8 +46,9 @@ POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 POSTGRES_EXPOSE=127.0.0.1:5432
 
-# Triton ML inference server
+# Triton ML inference server & TF Serving
 TRITON_HOST=triton
+TF_SERVING_HOST=tf_serving
 
 # InfluxDB
 INFLUXDB_HOST=
@@ -62,15 +63,12 @@ INFLUXDB_AUTH_TOKEN=
 # MONGO_URI=mongodb://mongodb.po_default:27017
 MONGO_URI=mongodb://mongodb:27017
 
+# Redis
+REDIS_HOST=redis
+
 # OpenFoodFacts API
 OFF_PASSWORD=
 OFF_USER=
 
 # Utils
-SENTRY_DSN=
-
-# Workers
-IPC_AUTHKEY=ipc
-IPC_HOST=workers
-IPC_PORT=6650
-WORKER_COUNT=8
+SENTRY_DSN=

.github/workflows/container-deploy-ml.yml

@@ -71,7 +71,7 @@ jobs:
           echo "COMPOSE_HTTP_TIMEOUT=120" >> .env
           echo "COMPOSE_PATH_SEPARATOR=;" >> .env
           echo "COMPOSE_PROJECT_NAME=robotoff-ml" >> .env
-          echo "COMPOSE_FILE=docker/ml.yml" >> .env
+          echo "COMPOSE_FILE=docker-compose.yml;docker/ml.yml" >> .env
           echo "RESTART_POLICY=always" >> .env
           echo "TRITON_EXPOSE_HTTP=8003" >> .env
 

.github/workflows/container-deploy.yml

@@ -111,10 +111,7 @@ jobs:
           # Set app variables
           echo "ROBOTOFF_INSTANCE=${{ env.ROBOTOFF_INSTANCE }}" >> .env
           echo "ROBOTOFF_DOMAIN=${{ env.ROBOTOFF_DOMAIN }}" >> .env
-          echo "IPC_AUTHKEY=${{ secrets.IPC_AUTHKEY }}" >> .env
-          echo "IPC_HOST=0.0.0.0" >> .env
-          echo "IPC_PORT=6650" >> .env
-          echo "WORKER_COUNT=8" >> .env
+          echo "REDIS_HOST=redis" >> .env
           echo "POSTGRES_HOST=postgres" >> .env
           echo "POSTGRES_DB=postgres" >> .env
           echo "POSTGRES_USER=postgres" >> .env

Makefile

@@ -55,7 +55,7 @@ up:
 	@echo "🥫 Building and starting containers …"
 	docker network create po_default || true  
 ifdef service
-	${DOCKER_COMPOSE} up -d ${service} 2>&1
+	${DOCKER_COMPOSE} up --remove-orphans -d ${service} 2>&1
 else
 	${DOCKER_COMPOSE} up -d 2>&1
 endif
@@ -173,26 +173,26 @@ health:
 i18n-compile:
 	@echo "🥫 Compiling translations …"
 # Note it's important to have --no-deps, to avoid launching a concurrent postgres instance
-	${DOCKER_COMPOSE} run --rm --entrypoint bash --no-deps workers -c "cd i18n && . compile.sh"
+	${DOCKER_COMPOSE} run --rm --entrypoint bash --no-deps worker_high -c "cd i18n && . compile.sh"
 
 unit-tests:
 	@echo "🥫 Running tests …"
 	# run tests in worker to have more memory
 	# also, change project name to run in isolation
-	${DOCKER_COMPOSE_TEST} run --rm workers poetry run pytest --cov-report xml --cov=robotoff tests/unit
+	${DOCKER_COMPOSE_TEST} run --rm worker_high poetry run pytest --cov-report xml --cov=robotoff tests/unit
 
 integration-tests:
 	@echo "🥫 Running integration tests …"
 	# run tests in worker to have more memory
 	# also, change project name to run in isolation
-	${DOCKER_COMPOSE_TEST} run --rm workers poetry run pytest -vv --cov-report xml --cov=robotoff --cov-append tests/integration
+	${DOCKER_COMPOSE_TEST} run --rm worker_high poetry run pytest -vv --cov-report xml --cov=robotoff --cov-append tests/integration
 	( ${DOCKER_COMPOSE_TEST} down -v || true )
 
 # interactive testings
 # usage: make pytest args='test/unit/my-test.py --pdb'
 pytest: guard-args
 	@echo "🥫 Running test: ${args} …"
-	${DOCKER_COMPOSE_TEST} run --rm workers poetry run pytest ${args}
+	${DOCKER_COMPOSE_TEST} run --rm worker_high poetry run pytest ${args}
 
 #------------#
 # Production #

doc/how-to-guides/deployment/maintenance.md

@@ -47,7 +47,7 @@ robotoff_api_1         /bin/sh -c /docker-entrypo ...   Up      0.0.0.0:5500->55
                                                                 /tcp                                
 robotoff_postgres_1    docker-entrypoint.sh postg ...   Up      127.0.0.1:5432->5432/tcp            
 robotoff_scheduler_1   /bin/sh -c /docker-entrypo ...   Up                                          
-robotoff_workers_1     /bin/sh -c /docker-entrypo ...   Up                              
+robotoff_worker_low_1     /bin/sh -c /docker-entrypo ...   Up                              
 ```
 
 ## Database backup and restore

doc/introduction/architecture.md

@@ -7,12 +7,12 @@ Robotoff is made of several services:
 - the public _API_ service
 - the _scheduler_, responsible for launching recurrent tasks (downloading new dataset, processing insights automatically,...) [^scheduler]
 - the _workers_, responsible for all long-lasting tasks
+- a _redis_ instance
 
-Communication between API and Workers happens through ipc events. [^ipc_events]
+Communication between API and workers happens through Redis DB using [rq](https://python-rq.org). [^worker_job]
 
 [^scheduler]: See `scheduler.run`
-
-[^ipc_events]: See `robotoff.workers.client` and `robotoff.workers.listener`
+[^worker_job]: See `robotoff.workers.queues` and `robotoff.workers.tasks`
 
 Robotoff allows to predict many information (also called _insights_), mostly from the product images or OCR.
 
@@ -58,7 +58,7 @@ Some insights with high confidence are applied automatically, 10 minutes after i
 
 Robotoff is also notified by Product Opener every time a product is updated or deleted [^product_update]. This is used to delete insights associated with deleted products, or to update them accordingly.
 
-[^product_update]: see `workers.tasks.product_updated` and `workers.tasks.delete_product_insights`
+[^product_update]: see `workers.tasks.product_updated` and `workers.tasks.delete_product_insights_job`
 [^annotate]: see `robotoff.insights.annotate`
 
 

docker-compose.yml

@@ -1,25 +1,23 @@
 version: "3.9"
 
-
-x-robotoff-base: &robotoff-base
+x-robotoff-base:
+  &robotoff-base
   restart: $RESTART_POLICY
   image: ghcr.io/openfoodfacts/robotoff:${TAG}
   volumes:
     - ./datasets:/opt/robotoff/datasets
     - ./tf_models:/opt/robotoff/tf_models
     - ./models:/opt/robotoff/models
 
-x-robotoff-base-env: &robotoff-base-env
+x-robotoff-base-env:
+  &robotoff-base-env
   ROBOTOFF_INSTANCE:
   ROBOTOFF_DOMAIN:
   ROBOTOFF_SCHEME:
   STATIC_OFF_DOMAIN:
   GUNICORN_NUM_WORKERS:
-  IPC_AUTHKEY:
-  IPC_HOST: workers
-  IPC_PORT:
-  WORKER_COUNT:
   ROBOTOFF_UPDATED_PRODUCT_WAIT:
+  REDIS_HOST:
   POSTGRES_HOST:
   POSTGRES_DB:
   POSTGRES_USER:
@@ -43,19 +41,33 @@ services:
     <<: *robotoff-base
     environment: *robotoff-base-env
     mem_limit: 2g
-    depends_on:
-      - workers
     ports:
       - "${ROBOTOFF_EXPOSE:-5500}:5500"
     networks:
       - webnet
 
-  workers:
+  worker_high:
     <<: *robotoff-base
-    command: poetry run robotoff-cli run workers
-    environment:
-      <<: *robotoff-base-env
-      REAL_TIME_IMAGE_PREDICTION: 1
+    deploy:
+      mode: replicated
+      replicas: 6
+    command: poetry run robotoff-cli run-worker robotoff-high
+    environment: *robotoff-base-env
+    depends_on:
+      - postgres
+    mem_limit: 8g
+    networks:
+      - webnet
+    extra_hosts:
+      - host.docker.internal:host-gateway
+
+  worker_low:
+    <<: *robotoff-base
+    deploy:
+      mode: replicated
+      replicas: 2
+    command: poetry run robotoff-cli run-worker robotoff-low robotoff-high
+    environment: *robotoff-base-env
     depends_on:
       - postgres
     mem_limit: 8g
@@ -67,7 +79,7 @@ services:
   scheduler:
     <<: *robotoff-base
     environment: *robotoff-base-env
-    command: poetry run robotoff-cli run scheduler
+    command: poetry run robotoff-cli run-scheduler
     mem_limit: 4g
     networks:
       - webnet
@@ -89,6 +101,19 @@ services:
     networks:
       - webnet
 
+  redis:
+    restart: $RESTART_POLICY
+    image: redis:7.0.5-alpine
+    volumes:
+      - redis-data:/data
+    environment:
+      REDIS_ARGS: --save 60 1000 --appendonly yes
+    mem_limit: 4g
+    ports:
+      - "${REDIS_EXPOSE:-127.0.0.1:6379}:6379"
+    networks:
+      - webnet
+
   elasticsearch:
     restart: $RESTART_POLICY
     image: raphael0202/elasticsearch
@@ -113,6 +138,8 @@ services:
 volumes:
   postgres-data:
   es-data:
+  redis-data:
+    name: ${COMPOSE_PROJECT_NAME:-robotoff}_redis-data
 
 networks:
   webnet:

docker/dev.yml

@@ -50,9 +50,20 @@ services:
         - robotoff.openfoodfacts.localhost
         - api
       webnet:
-  workers:
+  worker_high:
     <<: *robotoff-dev
     <<: *networks-productopener-local
+    deploy:
+      mode: replicated
+      # Only 1 replica is easier to deal with for local dev
+      replicas: 1
+  worker_low:
+    <<: *robotoff-dev
+    <<: *networks-productopener-local
+    deploy:
+      mode: replicated
+      # Only 1 replica is easier to deal with for local dev
+      replicas: 1
   scheduler:
     <<: *networks-productopener-local
     <<: *robotoff-dev

docker/ml.yml

@@ -8,7 +8,7 @@ services:
       - 8501:8501
       - 8500:8500
     volumes:
-      - ../tf_models:/models
+      - ./tf_models:/models
     entrypoint: "tensorflow_model_server --port=8500 --rest_api_port=8501 --model_config_file=/models/models.config"
     mem_limit: 10g
     networks:
@@ -28,11 +28,8 @@ services:
       - ${TRITON_EXPOSE_GRPC:-8001}:8001
       - ${TRITON_EXPOSE_METRICS:-8002}:8002
     volumes:
-      - ../models:/models
+      - ./models:/models
     entrypoint: "tritonserver --model-repository=/models"
     mem_limit: 10g
     networks:
       - webnet
-
-networks:
-  webnet: