From b817c701a62146a5ab0f57545803b2bc3ace2347 Mon Sep 17 00:00:00 2001 From: Halidu Abubakar Date: Tue, 6 Jun 2023 22:01:19 +0100 Subject: [PATCH 1/7] KLS-745 Prevent PII and Secrets from making their way into Great repositories --- .pre-commit-config.yaml | 52 +++++++++++++++++++++++++++++++++++++++++ pii-secret-exclude.txt | 0 2 files changed, 52 insertions(+) create mode 100644 .pre-commit-config.yaml create mode 100644 pii-secret-exclude.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..8beb5d53 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,52 @@ +repos: + - repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black + # Config for black lives in pyproject.toml + - repo: https://github.com/asottile/blacken-docs + rev: 1.13.0 + hooks: + - id: blacken-docs + additional_dependencies: [black==23.1.0] + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + - repo: https://github.com/pycqa/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + args: ["--markdown-linebreak-ext=md,markdown"] + - id: end-of-file-fixer + - id: check-yaml + - id: check-ast + - id: fix-byte-order-marker + - id: check-merge-conflict + - id: debug-statements + - id: detect-private-key + - id: detect-aws-credentials + args: ["--allow-missing-credentials"] + - repo: https://github.com/uktrade/pii-secret-check-hooks + rev: 0.0.0.35 + hooks: + - id: pii_secret_filename + files: '' + language: python + args: [exclude] + pass_filenames: true + require_serial: true + - id: pii_secret_file_content + files: '' + language: python + args: [--exclude=pii-secret-exclude.txt] + pass_filenames: true + require_serial: true + - id: hooks_version_check + name: Checking local hooks against latest release + verbose: true + require_serial: true diff --git a/pii-secret-exclude.txt b/pii-secret-exclude.txt new file mode 100644 index 00000000..e69de29b From 74f9b6445d8a46e39ff6795998dfe65f1d250fcd Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 14 Jun 2023 11:41:21 +0100 Subject: [PATCH 2/7] Move export_readiness modela --- .../0031_region_tag_industrytag_country.py | 46 +++++++++++++ core/migrations/0032_auto_20230614_0954.py | 43 +++++++++++++ core/snippets.py | 64 +++++++++++++++++++ .../models/great_international.py | 2 +- tests/export_readiness/test_snippets.py | 2 +- 5 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 core/migrations/0031_region_tag_industrytag_country.py create mode 100644 core/migrations/0032_auto_20230614_0954.py create mode 100644 core/snippets.py diff --git a/core/migrations/0031_region_tag_industrytag_country.py b/core/migrations/0031_region_tag_industrytag_country.py new file mode 100644 index 00000000..9cf06fbf --- /dev/null +++ b/core/migrations/0031_region_tag_industrytag_country.py @@ -0,0 +1,46 @@ +# Generated by Django 4.1.9 on 2023-06-14 09:39 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('wagtailimages', '0024_index_image_file_hash'), + ('core', '0030_greatmedia'), + ] + + operations = [ + migrations.CreateModel( + name='Region', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100, unique=True)), + ], + ), + migrations.CreateModel( + name='Tag', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100, unique=True)), + ], + ), + migrations.CreateModel( + name='IndustryTag', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100, unique=True)), + ('icon', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailimages.image')), + ], + ), + migrations.CreateModel( + name='Country', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100, unique=True)), + ('slug', models.CharField(max_length=100, unique=True)), + ('region', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.region')), + ], + ), + ] diff --git a/core/migrations/0032_auto_20230614_0954.py b/core/migrations/0032_auto_20230614_0954.py new file mode 100644 index 00000000..69497842 --- /dev/null +++ b/core/migrations/0032_auto_20230614_0954.py @@ -0,0 +1,43 @@ +# Generated by Django 4.1.9 on 2023-06-14 09:54 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0031_region_tag_industrytag_country'), + ] + + def copy_export_readiness_data(apps, schema_editor): + ExportReadinessTag = apps.get_model("export_readiness", "Tag") + ExportReadinessIndustryTag = apps.get_model("export_readiness", "IndustryTag") + ExportReadinessRegion = apps.get_model("export_readiness", "Region") + ExportReadinessCountry = apps.get_model("export_readiness", "Country") + + CoreTag = apps.get_model("core", "Tag") + CoreIndustryTag = apps.get_model("core", "IndustryTag") + CoreRegion = apps.get_model("core", "Region") + CoreCountry = apps.get_model("core", "Country") + + CoreRegion.objects.bulk_create(ExportReadinessRegion.objects.all()) + CoreTag.objects.bulk_create(ExportReadinessTag.objects.all()) + CoreCountry.objects.bulk_create(ExportReadinessCountry.objects.all()) + CoreIndustryTag.objects.bulk_create(ExportReadinessIndustryTag.objects.all()) + + + def delete_export_readiness_data(apps, schema_editor): + Tag = apps.get_model("core", "Tag") + IndustryTag = apps.get_model("core", "IndustryTag") + Region = apps.get_model("core", "Region") + Country = apps.get_model("core", "Country") + + Country.objects.all().delete() + IndustryTag.objects.all().delete() + Region.objects.all().delete() + Tag.objects.all().delete() + + + operations = [ + migrations.RunPython(copy_export_readiness_data, delete_export_readiness_data), + ] diff --git a/core/snippets.py b/core/snippets.py new file mode 100644 index 00000000..086faf8e --- /dev/null +++ b/core/snippets.py @@ -0,0 +1,64 @@ +from django.db import models + +from wagtail.admin.panels import FieldPanel +from wagtail.snippets.models import register_snippet + + +@register_snippet +class Tag(models.Model): + name = models.CharField(max_length=100, unique=True) + + def __str__(self): + return self.name + + panels = [ + FieldPanel('name') + ] + + +@register_snippet +class IndustryTag(models.Model): + name = models.CharField(max_length=100, unique=True) + icon = models.ForeignKey( + 'wagtailimages.Image', + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name='+' + ) + + def __str__(self): + return self.name + + panels = [ + FieldPanel('name'), + FieldPanel('icon') + ] + + +@register_snippet +class Region(models.Model): + name = models.CharField(max_length=100, unique=True) + + def __str__(self): + return self.name + + panels = [ + FieldPanel('name') + ] + + +@register_snippet +class Country(models.Model): + name = models.CharField(max_length=100, unique=True) + slug = models.CharField(max_length=100, unique=True) + region = models.ForeignKey(Region, null=True, blank=True, on_delete=models.SET_NULL) + + def __str__(self): + return self.name + + panels = [ + FieldPanel('name'), + FieldPanel('slug'), + FieldPanel('region') + ] diff --git a/great_international/models/great_international.py b/great_international/models/great_international.py index d37f2b8a..f31f8938 100644 --- a/great_international/models/great_international.py +++ b/great_international/models/great_international.py @@ -9,7 +9,7 @@ from core.mixins import ServiceHomepageMixin from core.model_fields import MarkdownField from core.models import WagtailAdminExclusivePageMixin -from export_readiness import snippets +from core import snippets from great_international.blocks import great_international as blocks from great_international.panels import great_international as panels from . import find_a_supplier as fas_models diff --git a/tests/export_readiness/test_snippets.py b/tests/export_readiness/test_snippets.py index 90767538..a378cc60 100644 --- a/tests/export_readiness/test_snippets.py +++ b/tests/export_readiness/test_snippets.py @@ -1,6 +1,6 @@ import pytest -from export_readiness.snippets import Tag +from core.snippets import Tag @pytest.mark.django_db From cb2d29fa3e3d1a6d1f34001d0dc39a6764f81976 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 14 Jun 2023 11:49:44 +0100 Subject: [PATCH 3/7] update requirements --- requirements.in | 4 ++-- requirements.txt | 27 ++++++++++++++++----------- requirements_test.txt | 33 +++++++++++++++++++-------------- test-reports/junit.xml | 1 + 4 files changed, 38 insertions(+), 27 deletions(-) create mode 100644 test-reports/junit.xml diff --git a/requirements.in b/requirements.in index 59eb5678..9e5dfff3 100644 --- a/requirements.in +++ b/requirements.in @@ -20,7 +20,7 @@ django-filter>=2.4.0 django-redis celery[redis] django-celery-beat==2.5.0 -kombu==5.2.3 +kombu==5.3.0 requests[security]>=2.31.0 markdown==2.* bleach==3.* @@ -38,4 +38,4 @@ gevent==22.10.* psycogreen==1.0.2 wagtailmedia==0.14.* cryptography==39.* -oauthlib==3.2.* \ No newline at end of file +oauthlib==3.2.* diff --git a/requirements.txt b/requirements.txt index d9518ea7..346b1612 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ beautifulsoup4==4.11.2 # via # directory-components # wagtail -billiard==3.6.4.0 +billiard==4.1.0 # via celery bleach==3.3.1 # via -r requirements.in @@ -32,7 +32,7 @@ botocore==1.27.96 # via # boto3 # s3transfer -celery[redis]==5.2.7 +celery[redis]==5.3.0 # via # -r requirements.in # django-celery-beat @@ -106,7 +106,7 @@ django-health-check==3.17.0 # via directory-healthcheck django-modelcluster==6.0 # via wagtail -django-modeltranslation==0.18.9 +django-modeltranslation==0.18.10 # via # -r requirements.in # wagtail-modeltranslation @@ -137,7 +137,9 @@ docopt==0.6.2 # num2words draftjs-exporter==2.1.7 # via wagtail -elastic-apm==6.15.1 +ecs-logging==2.0.2 + # via elastic-apm +elastic-apm==6.16.2 # via -r requirements.in et-xmlfile==1.1.0 # via openpyxl @@ -159,7 +161,7 @@ jmespath==1.0.1 # botocore jsonschema==3.2.0 # via directory-components -kombu==5.2.3 +kombu==5.3.0 # via # -r requirements.in # celery @@ -202,12 +204,12 @@ python-crontab==2.7.1 python-dateutil==2.8.2 # via # botocore + # celery # python-crontab pytube==9.2.2 # via -r requirements.in pytz==2023.3 # via - # celery # django-modelcluster # django-timezone-field # djangorestframework @@ -226,7 +228,7 @@ requests-oauthlib==1.3.1 # via django-staff-sso-client s3transfer==0.6.1 # via boto3 -sentry-sdk==1.24.0 +sentry-sdk==1.25.1 # via -r requirements.in sigauth==5.2.0 # via -r requirements.in @@ -245,15 +247,18 @@ soupsieve==2.4.1 # via beautifulsoup4 sqlparse==0.4.4 # via django -telepath==0.3 +telepath==0.3.1 # via wagtail -typing-extensions==4.6.2 +typing-extensions==4.6.3 # via # asgiref # dj-database-url # django-modeltranslation + # kombu tzdata==2023.3 - # via django-celery-beat + # via + # celery + # django-celery-beat urllib3==1.26.16 # via # -r requirements.in @@ -275,7 +280,7 @@ wagtail==4.1.5 # wagtailmedia wagtail-modeltranslation==0.13.0 # via -r requirements.in -wagtailmedia==0.14.1 +wagtailmedia==0.14.2 # via -r requirements.in wcwidth==0.2.6 # via prompt-toolkit diff --git a/requirements_test.txt b/requirements_test.txt index 0755da61..5586915f 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -20,7 +20,7 @@ beautifulsoup4==4.11.2 # via # directory-components # wagtail -billiard==3.6.4.0 +billiard==4.1.0 # via celery bleach==3.3.1 # via -r requirements.in @@ -34,7 +34,7 @@ botocore==1.27.96 # s3transfer build==0.10.0 # via pip-tools -celery[redis]==5.2.7 +celery[redis]==5.3.0 # via # -r requirements.in # django-celery-beat @@ -119,7 +119,7 @@ django-health-check==3.17.0 # via directory-healthcheck django-modelcluster==6.0 # via wagtail -django-modeltranslation==0.18.9 +django-modeltranslation==0.18.10 # via # -r requirements.in # wagtail-modeltranslation @@ -151,7 +151,9 @@ docopt==0.6.2 # num2words draftjs-exporter==2.1.7 # via wagtail -elastic-apm==6.15.1 +ecs-logging==2.0.2 + # via elastic-apm +elastic-apm==6.16.2 # via -r requirements.in et-xmlfile==1.1.0 # via openpyxl @@ -161,7 +163,7 @@ factory-boy==2.12.0 # via # -r requirements_test.in # wagtail-factories -faker==18.9.0 +faker==18.10.1 # via factory-boy flake8==6.0.0 # via -r requirements_test.in @@ -191,7 +193,7 @@ jmespath==1.0.1 # botocore jsonschema==3.2.0 # via directory-components -kombu==5.2.3 +kombu==5.3.0 # via # -r requirements.in # celery @@ -245,7 +247,7 @@ pyproject-hooks==1.0.0 # via build pyrsistent==0.19.3 # via jsonschema -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements_test.in # pytest-codecov @@ -267,6 +269,7 @@ python-crontab==2.7.1 python-dateutil==2.8.2 # via # botocore + # celery # faker # freezegun # python-crontab @@ -274,7 +277,6 @@ pytube==9.2.2 # via -r requirements.in pytz==2023.3 # via - # celery # django-modelcluster # django-timezone-field # djangorestframework @@ -292,13 +294,13 @@ requests[security]==2.31.0 # requests-mock # requests-oauthlib # wagtail -requests-mock==1.10.0 +requests-mock==1.11.0 # via -r requirements_test.in requests-oauthlib==1.3.1 # via django-staff-sso-client s3transfer==0.6.1 # via boto3 -sentry-sdk==1.24.0 +sentry-sdk==1.25.1 # via -r requirements.in sigauth==5.2.0 # via -r requirements.in @@ -323,7 +325,7 @@ sqlparse==0.4.4 # via # django # django-debug-toolbar -telepath==0.3 +telepath==0.3.1 # via wagtail termcolor==2.3.0 # via pytest-sugar @@ -333,13 +335,16 @@ tomli==2.0.1 # coverage # pyproject-hooks # pytest -typing-extensions==4.6.2 +typing-extensions==4.6.3 # via # asgiref # dj-database-url # django-modeltranslation + # kombu tzdata==2023.3 - # via django-celery-beat + # via + # celery + # django-celery-beat urllib3==1.26.16 # via # -r requirements.in @@ -364,7 +369,7 @@ wagtail-factories==2.0.1 # via -r requirements_test.in wagtail-modeltranslation==0.13.0 # via -r requirements.in -wagtailmedia==0.14.1 +wagtailmedia==0.14.2 # via -r requirements.in wcwidth==0.2.6 # via prompt-toolkit diff --git a/test-reports/junit.xml b/test-reports/junit.xml new file mode 100644 index 00000000..dad50e38 --- /dev/null +++ b/test-reports/junit.xml @@ -0,0 +1 @@ + \ No newline at end of file From cd3ed88ef6c7b5831ce421ce25ff7200005d2bc0 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 14 Jun 2023 11:52:41 +0100 Subject: [PATCH 4/7] removed test-reports~ --- test-reports/junit.xml | 1 - 1 file changed, 1 deletion(-) delete mode 100644 test-reports/junit.xml diff --git a/test-reports/junit.xml b/test-reports/junit.xml deleted file mode 100644 index dad50e38..00000000 --- a/test-reports/junit.xml +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file From ed2ea98e17b3ac9431a5ba6c7f57235a2c230522 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 14 Jun 2023 14:08:06 +0100 Subject: [PATCH 5/7] fix import --- core/upstream_serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/upstream_serializers.py b/core/upstream_serializers.py index 57c74325..0027bafa 100644 --- a/core/upstream_serializers.py +++ b/core/upstream_serializers.py @@ -13,7 +13,7 @@ from core import helpers from core.cache import SERVICE_NAMES_TO_ROOT_PATHS -from export_readiness.snippets import Tag +from core.snippets import Tag class AbstractFieldSerializer(abc.ABC): From 48984b6149bca45687ee5cb1cefd09396d217da1 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 14 Jun 2023 15:43:25 +0100 Subject: [PATCH 6/7] fix factories --- tests/export_readiness/factories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/export_readiness/factories.py b/tests/export_readiness/factories.py index 3160e926..c4763d71 100644 --- a/tests/export_readiness/factories.py +++ b/tests/export_readiness/factories.py @@ -2,7 +2,7 @@ import factory.fuzzy import wagtail_factories -from export_readiness import snippets +from core import snippets class TagFactory(factory.django.DjangoModelFactory): From 2d861b5f55eaa2ba0a8d0d8653cd99ee763b9211 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Thu, 15 Jun 2023 13:24:28 +0100 Subject: [PATCH 7/7] Delete Export Readiness Models --- conf/settings.py | 2 +- ...region_remove_industrytag_icon_and_more.py | 38 +++++++++++ export_readiness/snippets.py | 64 ------------------- ...ationalarticlelistingpage_tags_and_more.py | 40 ++++++++++++ 4 files changed, 79 insertions(+), 65 deletions(-) create mode 100644 export_readiness/migrations/0082_remove_country_region_remove_industrytag_icon_and_more.py delete mode 100644 export_readiness/snippets.py create mode 100644 great_international/migrations/0161_alter_internationalarticlelistingpage_tags_and_more.py diff --git a/conf/settings.py b/conf/settings.py index 7a4a7fc8..740f5592 100644 --- a/conf/settings.py +++ b/conf/settings.py @@ -85,9 +85,9 @@ 'storages', 'rest_framework', 'wagtailmedia', - 'export_readiness.apps.GreatDomesticConfig', 'great_international.apps.GreatInternationalConfig', 'components.apps.ComponentsConfig', + 'export_readiness.apps.GreatDomesticConfig', 'django_filters', 'authbroker_client', 'django_celery_beat', diff --git a/export_readiness/migrations/0082_remove_country_region_remove_industrytag_icon_and_more.py b/export_readiness/migrations/0082_remove_country_region_remove_industrytag_icon_and_more.py new file mode 100644 index 00000000..00753c70 --- /dev/null +++ b/export_readiness/migrations/0082_remove_country_region_remove_industrytag_icon_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 4.1.9 on 2023-06-14 22:30 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('great_international', '0160_auto_20230505_1933'), + ('export_readiness', '0081_auto_20211124_1053'), + ('core', '0032_auto_20230614_0954'), + ('great_international', '0161_alter_internationalarticlelistingpage_tags_and_more'), + + + ] + + operations = [ + migrations.RemoveField( + model_name='country', + name='region', + ), + migrations.RemoveField( + model_name='industrytag', + name='icon', + ), + migrations.DeleteModel( + name='Tag', + ), + migrations.DeleteModel( + name='Country', + ), + migrations.DeleteModel( + name='IndustryTag', + ), + migrations.DeleteModel( + name='Region', + ), + ] diff --git a/export_readiness/snippets.py b/export_readiness/snippets.py deleted file mode 100644 index 086faf8e..00000000 --- a/export_readiness/snippets.py +++ /dev/null @@ -1,64 +0,0 @@ -from django.db import models - -from wagtail.admin.panels import FieldPanel -from wagtail.snippets.models import register_snippet - - -@register_snippet -class Tag(models.Model): - name = models.CharField(max_length=100, unique=True) - - def __str__(self): - return self.name - - panels = [ - FieldPanel('name') - ] - - -@register_snippet -class IndustryTag(models.Model): - name = models.CharField(max_length=100, unique=True) - icon = models.ForeignKey( - 'wagtailimages.Image', - null=True, - blank=True, - on_delete=models.SET_NULL, - related_name='+' - ) - - def __str__(self): - return self.name - - panels = [ - FieldPanel('name'), - FieldPanel('icon') - ] - - -@register_snippet -class Region(models.Model): - name = models.CharField(max_length=100, unique=True) - - def __str__(self): - return self.name - - panels = [ - FieldPanel('name') - ] - - -@register_snippet -class Country(models.Model): - name = models.CharField(max_length=100, unique=True) - slug = models.CharField(max_length=100, unique=True) - region = models.ForeignKey(Region, null=True, blank=True, on_delete=models.SET_NULL) - - def __str__(self): - return self.name - - panels = [ - FieldPanel('name'), - FieldPanel('slug'), - FieldPanel('region') - ] diff --git a/great_international/migrations/0161_alter_internationalarticlelistingpage_tags_and_more.py b/great_international/migrations/0161_alter_internationalarticlelistingpage_tags_and_more.py new file mode 100644 index 00000000..3c96dcb5 --- /dev/null +++ b/great_international/migrations/0161_alter_internationalarticlelistingpage_tags_and_more.py @@ -0,0 +1,40 @@ +# Generated by Django 4.1.9 on 2023-06-14 22:30 + +from django.db import migrations +import modelcluster.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0032_auto_20230614_0954'), + ('great_international', '0160_auto_20230505_1933'), + ] + + operations = [ + migrations.AlterField( + model_name='internationalarticlelistingpage', + name='tags', + field=modelcluster.fields.ParentalManyToManyField(blank=True, to='core.tag'), + ), + migrations.AlterField( + model_name='internationalarticlepage', + name='tags', + field=modelcluster.fields.ParentalManyToManyField(blank=True, to='core.tag'), + ), + migrations.AlterField( + model_name='internationalcampaignpage', + name='tags', + field=modelcluster.fields.ParentalManyToManyField(blank=True, to='core.tag'), + ), + migrations.AlterField( + model_name='internationalinvestmentsectorpage', + name='tags', + field=modelcluster.fields.ParentalManyToManyField(blank=True, to='core.tag'), + ), + migrations.AlterField( + model_name='internationaltopiclandingpage', + name='tags', + field=modelcluster.fields.ParentalManyToManyField(blank=True, to='core.tag'), + ), + ]