Centralised server cache (#773)

* Added --no-recommends flag to apt-get install (#766)

* initial commit

* add cache configuration logic

* remove prints

* Add cache configuration tests

* update settings with cache config

* add new cache env fixtures

* add cache config generation helper

* remove unused imports

* copy updates

* add comment and type hint

* removed redis replication group id var

* add print statement

* handle member var with json loads

* update test fixture to use list

* update domain str length

---------

Co-authored-by: Murdo <[email protected]>

core/helpers.py

@@ -0,0 +1,42 @@
+import json
+import os
+from typing import Any
+
+
+def generate_cache_configuration() -> dict[str, Any]:
+    """
+    Generates appropriate cache configuration for the given environment
+    """
+    cache = {}
+    cache["BACKEND"] = "django.core.cache.backends.locmem.LocMemCache"
+
+    # Utilising Redis in Non local development environments
+    if (
+        os.environ.get("REDIS_AUTH_TOKEN")
+        and os.environ.get("REDIS_PRIMARY_ENDPOINT_ADDRESS")
+        and os.environ.get("REDIS_MEMBER_CLUSTERS")
+    ):
+        REDIS_DB_VALUE: int = 0
+        cache["BACKEND"] = "django.core.cache.backends.redis.RedisCache"
+
+        location: list[str] = []
+        location.append(
+            f"rediss://:{os.environ.get('REDIS_AUTH_TOKEN')}@{os.environ.get('REDIS_PRIMARY_ENDPOINT_ADDRESS')}/{REDIS_DB_VALUE}"  # noqa: E501
+        )
+
+        domain: str = ".".join(
+            os.environ.get("REDIS_PRIMARY_ENDPOINT_ADDRESS", "").split(".")[1:]
+        )  # 'iwfvzo.euw2.cache.amazonaws.com'
+
+        hosts: list[str] = json.loads(
+            os.environ.get("REDIS_MEMBER_CLUSTERS", "")
+        )  # ["cp-f05ff2dca7d81952-001","cp-f05ff2dca7d81952-002"]
+
+        for host in hosts:
+            location.append(
+                f"rediss://:{os.environ.get('REDIS_AUTH_TOKEN')}@{host}.{domain}/{REDIS_DB_VALUE}"  # noqa: E501
+            )
+
+        cache["LOCATION"] = location
+
+    return {"default": cache}

core/middleware.py

@@ -3,7 +3,6 @@
 from data_platform_catalogue.client.exceptions import ConnectivityError
 from django.conf import settings
 from django.core.exceptions import BadRequest
-from django.http import Http404
 from django.shortcuts import render
 from django.utils.translation import gettext as _
 

core/settings.py

@@ -6,6 +6,8 @@
 import sentry_sdk
 from dotenv import load_dotenv
 
+from .helpers import generate_cache_configuration
+
 TRUTHY_VALUES = ["True", "true", "T", "1"]
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -194,11 +196,8 @@
     },
 }
 
-CACHES = {
-    "default": {
-        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
-    }
-}
+# Cache Configuration
+CACHES = generate_cache_configuration()
 
 ANALYTICS_ID: str = os.environ.get("ANALYTICS_ID", "")
 ENABLE_ANALYTICS: bool = (

deployments/templates/deployment.yml

@@ -99,6 +99,21 @@ spec:
                 secretKeyRef:
                   name: rds-postgresql-instance-output
                   key: rds_instance_address
+            - name: REDIS_PRIMARY_ENDPOINT_ADDRESS
+              valueFrom:
+                secretKeyRef:
+                  name: fmd-redis-secrets
+                  key: primary_endpoint_address
+            - name: REDIS_AUTH_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: fmd-redis-secrets
+                  key: auth_token
+            - name: REDIS_MEMBER_CLUSTERS
+              valueFrom:
+                secretKeyRef:
+                  name: fmd-redis-secrets
+                  key: member_clusters
           securityContext:
             readOnlyRootFilesystem: true
             allowPrivilegeEscalation: false

locale/en/LC_MESSAGES/django.po

@@ -5,7 +5,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: Find MoJ data\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-08-28 14:34+0100\n"
+"POT-Creation-Date: 2024-09-05 19:37+0100\n"
 "Language: en\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -20,14 +20,14 @@ msgstr "Catalogue service unavailable"
 msgid "Bad request"
 msgstr "Bad request"
 
-#: core/middleware.py:42 templates/base/error/page_not_found.html:5
-msgid "Page not found"
-msgstr "Page not found"
-
-#: core/middleware.py:49
+#: core/middleware.py:42
 msgid "There is a problem with this service"
 msgstr "There is a problem with this service"
 
+#: core/views.py:9 templates/base/error/page_not_found.html:5
+msgid "Page not found"
+msgstr "Page not found"
+
 #: feedback/models.py:16
 msgid "Satisfaction survey"
 msgstr "Satisfaction survey"
@@ -148,7 +148,7 @@ msgstr "Available on"
 msgid "Qualified name"
 msgstr "Qualified name"
 
-#: home/views.py:27 templates/base/navigation.html:64
+#: home/views.py:30 templates/base/navigation.html:64
 msgid "Home"
 msgstr "Home"
 
@@ -192,7 +192,7 @@ msgstr ""
 msgid "Support links"
 msgstr "Support links"
 
-#: templates/base/footer.html:10 templates/cookies.html:10
+#: templates/base/footer.html:10 templates/cookies.html:11
 msgid "Cookies"
 msgstr "Cookies"
 

pyproject.toml

@@ -22,6 +22,7 @@ django-azure-auth = "1.3.0"
 django-waffle = "^4.1.0"
 psycopg = "^3.2.1"
 psycopg-binary = "^3.2.1"
+redis = {extras = ["hiredis"], version = "^5.0.8"}
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.8.0"

tests/conftest.py

@@ -601,3 +601,23 @@ def dataset_with_parent(mock_catalogue) -> dict[str, Any]:
         "parent_entity": container,
         "table_metadata": table_metadata,
     }
+
+
+@pytest.fixture(scope="function")
+def set_redis_cache_env(monkeypatch):
+    monkeypatch.setenv("REDIS_AUTH_TOKEN", "testredistoken")
+    monkeypatch.setenv(
+        "REDIS_PRIMARY_ENDPOINT_ADDRESS",
+        "master.cp-12345.iwfvzo.euw2.cache.amazonaws.com",
+    )
+    monkeypatch.setenv(
+        "REDIS_MEMBER_CLUSTERS",
+        '["cp-f05ff2dca7d81952-001", "cp-f05ff2dca7d81952-002"]',
+    )
+
+
+@pytest.fixture(scope="function")
+def unset_redis_cache_env(monkeypatch):
+    monkeypatch.delenv("REDIS_AUTH_TOKEN", raising=False)
+    monkeypatch.delenv("REDIS_PRIMARY_ENDPOINT_ADDRESS", raising=False)
+    monkeypatch.delenv("REDIS_MEMBER_CLUSTERS", raising=False)

tests/core/test_cache_configuration.py

@@ -0,0 +1,20 @@
+from core.helpers import generate_cache_configuration
+
+
+def test_redis_cache_for_non_local_development(set_redis_cache_env):
+    cache = generate_cache_configuration()
+    location = cache["default"]["LOCATION"]
+    assert len(location) == 3
+    assert (
+        location[0]
+        == "rediss://:[email protected]/0"
+    )
+    assert cache["default"]["BACKEND"] == "django.core.cache.backends.redis.RedisCache"
+
+
+def test_memcache_for_local_development(unset_redis_cache_env):
+    cache = generate_cache_configuration()
+    assert "LOCATION" not in cache["default"]
+    assert (
+        cache["default"]["BACKEND"] == "django.core.cache.backends.locmem.LocMemCache"
+    )

tests/core/test_middleware.py

@@ -2,7 +2,6 @@
 
 from data_platform_catalogue.client.exceptions import ConnectivityError
 from django.core.exceptions import BadRequest
-from django.http import Http404
 
 from core.middleware import CustomErrorMiddleware