Merge pull request #841 from uktrade/develop

Staging release

CHANGELOG.md

@@ -3,6 +3,14 @@
 ## Pre-release
 
 ### Implemented enhancements
+- No ticket - Improve caching of markets endpoint
+
+## Fixed bugs
+- No ticket - Upgrade django to fix security vulnerability
+
+## [2020.05.28](https://github.com/uktrade/directory-cms/releases/tag/2020.05.28)
+[Full Changelog](https://github.com/uktrade/directory-cms/compare/2020.05.26...2020.05.28)
+
 ## Fixed bugs
 - No ticket - Upgraded dependencies to fix security vulnerability
 - No ticket - Expedite non-success response during cache miss

core/cache.py

@@ -1,4 +1,6 @@
+import collections
 import hashlib
+import itertools
 from urllib.parse import urlencode
 
 from directory_constants import cms, slugs
@@ -14,6 +16,7 @@
 
 from core.serializer_mapping import MODELS_SERIALIZERS_MAPPING
 from conf.celery import app
+from export_readiness.models import CountryGuidePage
 
 
 ROOT_PATHS_TO_SERVICE_NAMES = {
@@ -274,6 +277,70 @@ def subscribe(cls):
         post_migrate.connect(receiver=cls.clear)
 
 
+class CountryPagesCache:
+    cache = cache
+
+    @staticmethod
+    def build_key(country, industry):
+        return f'countryguide_{country}{industry}'
+
+    @classmethod
+    def set(cls, data, country=None, industry=None,):
+        key = cls.build_key(country=country, industry=industry)
+        cls.cache.set(key, data, timeout=settings.API_CACHE_EXPIRE_SECONDS)
+
+    @classmethod
+    def get_many(cls, industries=[None], countries=[None]):
+        keys = [cls.build_key(*args) for args in itertools.product(countries, industries)]
+        pages = {}
+        # making the values returned distint
+        for records in cls.cache.get_many(keys).values():
+            for record in records:
+                pages[record['id']] = record
+
+        return list(pages.values())
+
+    @classmethod
+    def transaction(cls):
+        class Transaction(cls):
+            cache = TransactionalCache()
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *args):
+                self.cache.commit()
+
+        return Transaction()
+
+
+class CountryPageCachePopulator:
+
+    @classmethod
+    def populate(cls, *args, **kwargs):
+        pages = collections.defaultdict(list)
+        serializer_class = MODELS_SERIALIZERS_MAPPING[CountryGuidePage]
+
+        # store the record four times: so can be filtered by country+industry, country, industry, or no filter
+        for page in CountryGuidePage.objects.select_related('country').live():
+            serializer = serializer_class(instance=page)
+
+            # allows retrieve with no filter
+            pages[(None, None)].append(serializer.data)
+            for industry in page.tags.values_list('name', flat=True):
+                if page.country:
+                    # allow retrieve with both filters
+                    pages[(industry, page.country.name)].append(serializer.data)
+                    # allow retrieve with country
+                    pages[(None, page.country.name)].append(serializer.data)
+                # allow retrieve with industry
+                pages[(industry, None)].append(serializer.data)
+
+        with CountryPagesCache.transaction() as country_cache:
+            for (industry, country), data in pages.items():
+                country_cache.set(data, country=country, industry=industry)
+
+
 class DatabaseCacheSubscriber:
 
     cache_populator = CachePopulator
@@ -307,3 +374,4 @@ def rebuild_all_cache():
     for page in Page.objects.live().specific():
         if page.__class__ in MODELS_SERIALIZERS_MAPPING and page.__class__ is not Page:
             CachePopulator.populate_async(page)
+    CountryPageCachePopulator.populate()

export_readiness/views.py

@@ -1,15 +1,11 @@
-import json
-
-from django.core.cache import cache
-from django.db.models import Q
 from django.utils.decorators import method_decorator
 from django.views.decorators.cache import cache_page
 from rest_framework.generics import RetrieveAPIView, ListAPIView
 from rest_framework.response import Response
 
 from conf.signature import SignatureCheckPermission
-from export_readiness import models, serializers, snippets
-
+from export_readiness import serializers, snippets
+from core.cache import CountryPagesCache
 
 THIRTY_MINUTES_IN_SECONDS = 30 * 60
 
@@ -31,45 +27,19 @@ def get(self, request, *args, **kwargs):
 
 
 class CountryPageListAPIView(ListAPIView):
-    queryset = models.CountryGuidePage.objects.all()
-    serializer_class = serializers.CountryGuidePageSerializer
     permission_classes = [SignatureCheckPermission]
     http_method_names = ('get', )
 
-    @property
-    def cache_key(self):
-        industry = self.request.GET.get('industry')
-        region = self.request.GET.get('region')
-        return f'countryguide_{industry}{region}'
-
-    def get_queryset(self):
-        queryset = models.CountryGuidePage.objects.filter(live=True)
-        industry = self.request.query_params.get('industry')
-        region = self.request.query_params.get('region')
-        if not industry and not region:
-            return queryset
-        q_industries = Q()
-        q_regions = Q()
-        if industry:
-            for value in industry.split(','):
-                q_industries |= Q(tags__name=value)
-        if region:
-            for value in region.split(','):
-                q_regions |= Q(country__region__name=value)
-        return queryset.filter(q_industries & q_regions).distinct()
-
     def get(self, *args, **kwargs):
-
-        def foo(response):
-            cache.set(key=self.cache_key, value=response.content, timeout=THIRTY_MINUTES_IN_SECONDS)
-
-        cached_content = cache.get(key=self.cache_key)
-        if cached_content:
-            response = Response(json.loads(cached_content), content_type='application/json')
-        else:
-            response = super().get(*args, **kwargs)
-            response.add_post_render_callback(foo)
-        return response
+        filters = {}
+        industries = self.request.GET.get('industry')
+        countries = self.request.GET.get('region')
+        if industries:
+            filters['industries'] = industries.split(',')
+        if countries:
+            filters['countries'] = countries.split(',')
+        cached_content = CountryPagesCache.get_many(**filters)
+        return Response(cached_content or [], content_type='application/json')
 
 
 class RegionsListAPIView(ListAPIView):

requirements.txt

@@ -4,16 +4,16 @@
 #
 #    pip-compile requirements.in
 #
-amqp==2.5.2               # via kombu
+amqp==2.6.0               # via kombu
 attrs==19.3.0             # via jsonschema
 beautifulsoup4==4.6.0     # via directory-components, wagtail
 billiard==3.6.3.0         # via celery
 bleach-whitelist==0.0.10  # via -r requirements.in
 bleach==3.1.5             # via -r requirements.in
 boto3==1.6.3              # via -r requirements.in
 botocore==1.9.23          # via boto3, s3transfer
-celery[redis]==4.4.2      # via -r requirements.in, django-celery-beat
-certifi==2020.4.5.1       # via elastic-apm, requests, sentry-sdk
+celery[redis]==4.4.5      # via -r requirements.in, django-celery-beat
+certifi==2020.4.5.2       # via elastic-apm, requests, sentry-sdk
 chardet==3.0.4            # via requests
 directory-components==20.3.1  # via -r requirements.in
 directory-constants==18.7.0  # via -r requirements.in, directory-components
@@ -33,24 +33,24 @@ django-staff-sso-client==1.0.1  # via -r requirements.in
 django-taggit==1.3.0      # via wagtail
 django-timezone-field==4.0  # via django-celery-beat
 django-treebeard==4.3.1   # via wagtail
-django==2.2.12            # via -r requirements.in, directory-components, directory-constants, directory-healthcheck, django-admin-ip-restrictor, django-celery-beat, django-filter, django-modeltranslation, django-redis, django-staff-sso-client, django-storages, django-taggit, django-timezone-field, django-treebeard, sigauth, wagtail
+django==2.2.13            # via -r requirements.in, directory-components, directory-constants, directory-healthcheck, django-admin-ip-restrictor, django-celery-beat, django-filter, django-modeltranslation, django-redis, django-staff-sso-client, django-storages, django-taggit, django-timezone-field, django-treebeard, sigauth, wagtail
 django_storages==1.7.1    # via -r requirements.in
 djangorestframework==3.9.4  # via -r requirements.in, sigauth, wagtail
 docopt==0.6.2             # via notifications-python-client, num2words
 docutils==0.16            # via botocore
 draftjs-exporter==2.1.7   # via wagtail
 elastic-apm==5.6.0        # via -r requirements.in
-future==0.18.2            # via notifications-python-client
+future==0.18.2            # via celery, notifications-python-client
 gevent==1.2.2             # via -r requirements.in
-greenlet==0.4.15          # via gevent
+greenlet==0.4.16          # via gevent
 gunicorn==19.5.0          # via -r requirements.in
 html2text==2018.1.9       # via -r requirements.in
 html5lib==1.0.1           # via wagtail
 idna==2.8                 # via requests
-importlib-metadata==1.6.0  # via kombu
+importlib-metadata==1.6.1  # via kombu
 jmespath==0.10.0          # via boto3, botocore
 jsonschema==3.0.1         # via directory-components
-kombu==4.6.8              # via -r requirements.in, celery
+kombu==4.6.10             # via -r requirements.in, celery
 markdown==2.6             # via -r requirements.in
 mohawk==0.3.4             # via sigauth
 monotonic==1.5            # via notifications-python-client
@@ -68,7 +68,7 @@ python-crontab==2.5.1     # via django-celery-beat
 python-dateutil==2.6.1    # via botocore, python-crontab
 pytube==9.2.2             # via -r requirements.in
 pytz==2020.1              # via celery, django, django-modelcluster, django-timezone-field, wagtail
-redis==3.5.2              # via celery, django-redis
+redis==3.5.3              # via celery, django-redis
 requests-oauthlib==1.3.0  # via django-staff-sso-client
 requests==2.21.0          # via -r requirements.in, notifications-python-client, requests-oauthlib, wagtail
 s3transfer==0.1.13        # via boto3

requirements_test.txt

@@ -4,16 +4,16 @@
 #
 #    pip-compile requirements_test.in
 #
-amqp==2.5.2               # via kombu
+amqp==2.6.0               # via kombu
 attrs==19.3.0             # via jsonschema, pytest
 beautifulsoup4==4.6.0     # via -r requirements_test.in, directory-components, wagtail
 billiard==3.6.3.0         # via celery
 bleach-whitelist==0.0.10  # via -r requirements.in
 bleach==3.1.5             # via -r requirements.in
 boto3==1.6.3              # via -r requirements.in
 botocore==1.9.23          # via boto3, s3transfer
-celery[redis]==4.4.2      # via -r requirements.in, django-celery-beat
-certifi==2020.4.5.1       # via elastic-apm, requests, sentry-sdk
+celery[redis]==4.4.5      # via -r requirements.in, django-celery-beat
+certifi==2020.4.5.2       # via elastic-apm, requests, sentry-sdk
 chardet==3.0.4            # via requests
 click==7.1.2              # via pip-tools
 coverage==5.1             # via coveralls, pytest-cov
@@ -37,7 +37,7 @@ django-staff-sso-client==1.0.1  # via -r requirements.in
 django-taggit==1.3.0      # via wagtail
 django-timezone-field==4.0  # via django-celery-beat
 django-treebeard==4.3.1   # via wagtail
-django==2.2.12            # via -r requirements.in, directory-components, directory-constants, directory-healthcheck, django-admin-ip-restrictor, django-celery-beat, django-debug-toolbar, django-filter, django-modeltranslation, django-redis, django-staff-sso-client, django-storages, django-taggit, django-timezone-field, django-treebeard, sigauth, wagtail
+django==2.2.13            # via -r requirements.in, directory-components, directory-constants, directory-healthcheck, django-admin-ip-restrictor, django-celery-beat, django-debug-toolbar, django-filter, django-modeltranslation, django-redis, django-staff-sso-client, django-storages, django-taggit, django-timezone-field, django-treebeard, sigauth, wagtail
 django_storages==1.7.1    # via -r requirements.in
 djangorestframework==3.9.4  # via -r requirements.in, sigauth, wagtail
 docopt==0.6.2             # via coveralls, notifications-python-client, num2words
@@ -48,17 +48,17 @@ factory-boy==2.12.0       # via -r requirements_test.in, wagtail-factories
 faker==4.1.0              # via factory-boy
 flake8==3.8.2             # via -r requirements_test.in
 freezegun==0.3.14         # via -r requirements_test.in
-future==0.18.2            # via notifications-python-client
+future==0.18.2            # via celery, notifications-python-client
 gevent==1.2.2             # via -r requirements.in
-greenlet==0.4.15          # via gevent
+greenlet==0.4.16          # via gevent
 gunicorn==19.5.0          # via -r requirements.in
 html2text==2018.1.9       # via -r requirements.in
 html5lib==1.0.1           # via wagtail
 idna==2.8                 # via requests
-importlib-metadata==1.6.0  # via flake8, kombu, pluggy, pytest
+importlib-metadata==1.6.1  # via flake8, kombu, pluggy, pytest
 jmespath==0.10.0          # via boto3, botocore
 jsonschema==3.0.1         # via directory-components
-kombu==4.6.8              # via -r requirements.in, celery
+kombu==4.6.10             # via -r requirements.in, celery
 markdown==2.6             # via -r requirements.in
 mccabe==0.6.1             # via flake8
 mohawk==0.3.4             # via sigauth
@@ -69,7 +69,7 @@ num2words==0.5.10         # via -r requirements.in
 oauthlib==3.1.0           # via requests-oauthlib
 packaging==20.4           # via bleach, pytest, pytest-sugar
 pillow==6.2.2             # via -r requirements.in, wagtail
-pip-tools==5.1.2          # via -r requirements_test.in
+pip-tools==5.2.0          # via -r requirements_test.in
 pluggy==0.13.1            # via pytest
 psycopg2==2.7.3.2         # via -r requirements.in
 py==1.8.1                 # via pytest
@@ -81,12 +81,13 @@ pyparsing==2.4.7          # via packaging
 pyrsistent==0.16.0        # via jsonschema
 pytest-cov==2.9.0         # via -r requirements_test.in
 pytest-django==3.9.0      # via -r requirements_test.in
-pytest==5.4.2             # via -r requirements_test.in, pytest-cov, pytest-django, pytest-sugar
+pytest-sugar==0.9.3       # via -r requirements_test.in
+pytest==5.4.3             # via -r requirements_test.in, pytest-cov, pytest-django, pytest-sugar
 python-crontab==2.5.1     # via django-celery-beat
 python-dateutil==2.6.1    # via botocore, faker, freezegun, python-crontab
 pytube==9.2.2             # via -r requirements.in
 pytz==2020.1              # via celery, django, django-modelcluster, django-timezone-field, wagtail
-redis==3.5.2              # via celery, django-redis
+redis==3.5.3              # via celery, django-redis
 requests-mock==1.8.0      # via -r requirements_test.in
 requests-oauthlib==1.3.0  # via django-staff-sso-client
 requests==2.21.0          # via -r requirements.in, coveralls, notifications-python-client, requests-mock, requests-oauthlib, wagtail
@@ -105,7 +106,7 @@ wagtail-factories==2.0.0  # via -r requirements_test.in
 wagtail-modeltranslation==0.10.13  # via -r requirements.in
 wagtail==2.7.3            # via -r requirements.in, wagtail-factories, wagtail-modeltranslation, wagtailmedia
 wagtailmedia==0.5.0       # via -r requirements.in
-wcwidth==0.1.9            # via pytest
+wcwidth==0.2.4            # via pytest
 webencodings==0.5.1       # via bleach, html5lib
 whitenoise==4.1.2         # via -r requirements.in
 willow==1.3               # via wagtail