working celery basics

Dockerfile

@@ -6,10 +6,13 @@ RUN apt update --fix-missing  \
     && apt install libpq-dev gcc -y  \
     && apt install build-essential -y --no-install-recommends
 
-RUN pip3 install pipenv
+ENV HOME /root
+ENV PYENV_ROOT $HOME/.pyenv
+ENV PATH $PYENV_ROOT/bin:$PATH
 
+RUN pip3 install pipenv
 ADD Pipfile* /app/
+RUN pipenv install --dev --deploy
 
-RUN pipenv sync --dev
 
 ADD . /app

Pipfile

@@ -40,7 +40,6 @@ setuptools = "~=65.5.1"
 celery = "~=5.2.7"
 redis = "~=4.0.2"
 
-
 [requires]
 python_version = "3.8"
 python_full_version = "3.8.17"

Procfile

@@ -1,4 +1,4 @@
 # See gunicorn.conf.py for more configuration.
 web: python manage.py migrate && gunicorn conf.wsgi:application
 worker: python manage.py process_tasks --log-std
-celeryworker: celery -A api.conf worker -l info
+celeryworker: celery -A conf worker -l info

README.md

@@ -9,7 +9,7 @@ Tasks are managed using this project: [Django Background Tasks](https://github.c
 We currently have two mechanisms for background tasks in lite;
 - django-background-tasks: `pipenv run ./manage.py process_tasks` will run all background tasks
 - celery: a celery container is running by default when using docker-compose.  If a working copy
-    "on the metal" without docker, run celery with `watchmedo auto-restart -d . -R -p '*.py' -- celery -A api.conf worker -l info`
+    "on the metal" without docker, run celery with `watchmedo auto-restart -d . -R -p '*.py' -- celery -A conf worker -l info`
 
 The entry point for configuring the tasks is defined here: `lite-hmrc/mail/apps.py`
 

conf/celery.py

@@ -3,7 +3,7 @@
 from celery import Celery
 
 # Set the default Django settings module for the 'celery' program.
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "api.conf.settings")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conf.settings")
 
 app = Celery("hmrc")
 

conf/settings.py

@@ -266,17 +266,6 @@
     if "aws-s3-bucket" not in VCAP_SERVICES:
         raise Exception("S3 Bucket not bound to environment")
 
-    aws_credentials = VCAP_SERVICES["aws-s3-bucket"][0]["credentials"]
-    AWS_ACCESS_KEY_ID = aws_credentials["aws_access_key_id"]
-    AWS_SECRET_ACCESS_KEY = aws_credentials["aws_secret_access_key"]
-    AWS_REGION = aws_credentials["aws_region"]
-    AWS_STORAGE_BUCKET_NAME = aws_credentials["bucket_name"]
-else:
-    AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID")
-    AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY")
-    AWS_REGION = env("AWS_REGION")
-    AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
-
 if "redis" in VCAP_SERVICES:
     REDIS_BASE_URL = VCAP_SERVICES["redis"][0]["credentials"]["uri"]
 else:

core/tests/test_celery_tasks.py

@@ -1,4 +1,4 @@
-from api.core import celery_tasks
+from core import celery_tasks
 from rest_framework.test import APITestCase
 
 class FlagsUpdateTest(APITestCase):

docker-compose.yml

@@ -14,6 +14,8 @@ services:
       - 5432
     ports:
       - 5432:5432
+    networks:
+      - lite
 
   lite-hmrc-intg:
     container_name: "lite-hmrc-intg"
@@ -30,12 +32,18 @@ services:
     expose:
       - 8000
     command: pipenv run ./manage.py runserver 0.0.0.0:8000
+    networks:
+      - lite
+
 
   mailhog:
     ports:
       - 8025:8025  # HTTP
       - 587:1025  # SMTP
     image: mailhog/mailhog
+    networks:
+      - lite
+
 
   celery:
     build: .
@@ -47,15 +55,38 @@ services:
       - redis
     depends_on:
       - lite-hmrc-intg
-    command: watchmedo auto-restart -d . -R -p '*.py' -- celery -A conf worker -l info
+    command: pipenv run watchmedo auto-restart -d . -R -p '*.py' -- celery -A conf worker -l info
+    networks:
+      - lite
+
+
+  celery-scheduler:
+    build: .
+    volumes:
+      - .:/app
+    env_file: .env
+    links:
+      - lite-hmrc-postgres
+      - redis
+    depends_on:
+      - lite-hmrc-intg
+    command:  pipenv run watchmedo auto-restart -d . -R -p '*.py' -- celery -A conf beat
+    networks:
+      - lite
+
 
   redis:
     image: "redis:5-alpine"
-    container_name: redis
+    container_name: hmrc-redis
     expose:
       - 6379
     ports:
       - 6379:6379
+    networks:
+      - lite
 
+networks:
+  lite:
+    external: true
 volumes:
   maildata:

docker.env

@@ -59,3 +59,4 @@ LITE_API_URL=https://lite.example.com
 AZURE_AUTH_CLIENT_ID=
 AZURE_AUTH_CLIENT_SECRET=
 AZURE_AUTH_TENANT_ID=
+REDIS_BASE_URL=redis://redis:6379

local.env

@@ -58,3 +58,4 @@ AZURE_AUTH_CLIENT_SECRET=<FROM_VAULT>
 AZURE_AUTH_TENANT_ID=<FROM_VAULT>
 
 SEND_REJECTED_EMAIL=True
+REDIS_BASE_URL=redis://localhost:6379