[#1515] Remove worker container from local environments

.fides/celery.toml

@@ -1,2 +1,3 @@
 event_queue_prefix = "fidesops_worker"
 task_default_queue = "fidesops"
+task_always_eager = true

docker-compose.worker.yml

@@ -0,0 +1,18 @@
+services:
+  worker:
+    image: ethyca/fides:local
+    command: fides worker
+    depends_on:
+      redis:
+        condition: service_started
+    restart: always
+    environment:
+      FIDES__CONFIG_PATH: ${FIDES__CONFIG_PATH:-/fides/.fides/fides.toml}
+      FIDES__TEST_MODE: "True"
+      FIDES__USER__ANALYTICS_OPT_OUT: "True"
+    volumes:
+      - type: bind
+        source: ./
+        target: /fides
+        read_only: False
+      - /fides/src/fides.egg-info

docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   fides:
     image: ethyca/fides:local
-    command: uvicorn --host 0.0.0.0 --port 8080 --reload fides.api.main:app
+    command: uvicorn --host 0.0.0.0 --port 8080 --reload --reload-dir src fides.api.main:app
     healthcheck:
       test: [ "CMD", "curl", "-f", "http://0.0.0.0:8080/health" ]
       interval: 20s
@@ -12,7 +12,7 @@ services:
     depends_on:
       fides-db:
         condition: service_healthy
-      worker:
+      redis:
         condition: service_started
     expose:
       - 8080
@@ -25,7 +25,6 @@ services:
       FIDES__CLI__SERVER_PORT: "8080"
       FIDES__DATABASE__SERVER: "fides-db"
       FIDES__DEV_MODE: "True"
-      FIDES__EXECUTION__WORKER_ENABLED: "True"
       FIDES__REDIS__ENABLED: "True"
       FIDES__TEST_MODE: "True"
       FIDES__USER__ANALYTICS_OPT_OUT: "True"
@@ -114,24 +113,6 @@ services:
     ports:
       - "0.0.0.0:6379:6379"
 
-  worker:
-    image: ethyca/fides:local
-    command: fides worker
-    depends_on:
-      redis:
-        condition: service_started
-    restart: always
-    environment:
-      FIDES__CONFIG_PATH: ${FIDES__CONFIG_PATH:-/fides/.fides/fides.toml}
-      FIDES__TEST_MODE: "True"
-      FIDES__USER__ANALYTICS_OPT_OUT: "True"
-    volumes:
-      - type: bind
-        source: ./
-        target: /fides
-        read_only: False
-      - /fides/src/fides.egg-info
-
 volumes:
   postgres: null
 

docs/fides/docs/installation/configuration.md

@@ -63,7 +63,6 @@ task_retry_backoff = 1
 subject_identity_verification_required = false
 task_retry_count = 0
 task_retry_delay = 1
-worker_enabled = false
 
 [admin_ui]
 enabled = true
@@ -142,7 +141,6 @@ The `fides.toml` file should specify the following variables:
 |`require_manual_request_approval` | bool | `False` | Whether privacy requests require explicit approval to execute. |
 |`masking_strict` | bool | `True` | If set to `True`, only use UPDATE requests to mask data. If `False`, Fides will use any defined DELETE or GDPR DELETE endpoints to remove PII, which may extend beyond the specific data categories that configured in your execution policy. |
 |`celery_config_path` | string | N/A | An optional override for the [Celery](#celery-configuration) configuration file path. |
-|`worker_enabled` | bool | `True` | By default, Fides uses a dedicated [Celery worker](#celery-configuration) to process privacy requests asynchronously. Setting `worker_enabled` to `False` will run the worker on the same node as the webserver. |
 
 #### User 
 

src/fides/api/main.py

@@ -2,7 +2,6 @@
 Contains the code that sets up the API.
 """
 import logging
-import subprocess
 from datetime import datetime, timezone
 from logging import WARNING
 from os import getenv
@@ -243,10 +242,6 @@ async def setup_server() -> None:
         )
     )
 
-    if not CONFIG.execution.worker_enabled:
-        logger.info("Starting worker...")
-        subprocess.Popen(["fides", "worker"])  # pylint: disable=consider-using-with
-
     setup_logging(
         CONFIG.logging.level,
         serialize=CONFIG.logging.serialization,

src/fides/ctl/core/config/execution_settings.py

@@ -13,7 +13,6 @@ class ExecutionSettings(FidesSettings):
     subject_identity_verification_required: bool = False
     require_manual_request_approval: bool = False
     masking_strict: bool = True
-    worker_enabled: bool = False
     celery_config_path: str = ".fides/celery.toml"
 
     class Config:

tests/ops/conftest.py

@@ -204,6 +204,11 @@ def integration_config():
     yield load_toml(["tests/ops/integration_test_config.toml"])
 
 
+@pytest.fixture(scope="session")
+def celery_config():
+    return {"task_always_eager": False}
+
+
 @pytest.fixture(autouse=True, scope="session")
 def celery_enable_logging():
     """Turns on celery output logs."""