From 39663d7f1ca3dc5e582cf5b4eaf13b3204fd87cf Mon Sep 17 00:00:00 2001 From: vasileios Date: Fri, 19 Jan 2024 16:37:15 +0100 Subject: [PATCH] [#3783] Added statistics view for the submitted forms to the admin The submissions exist in the db but are cleaned periodically. We added a model which keeps track of the submitted forms with a counter. We keep the form name in case the form is deleted. --- .../fixtures/default_admin_index.json | 4 + src/openforms/forms/admin/__init__.py | 1 + src/openforms/forms/admin/form_statistics.py | 33 +++++ .../forms/migrations/0104_formstatistics.py | 72 ++++++++++ src/openforms/forms/models/__init__.py | 2 + src/openforms/forms/models/form_statistics.py | 40 ++++++ src/openforms/forms/tests/test_statistics.py | 125 ++++++++++++++++++ src/openforms/submissions/api/mixins.py | 6 +- src/openforms/submissions/signals.py | 23 +++- 9 files changed, 303 insertions(+), 3 deletions(-) create mode 100644 src/openforms/forms/admin/form_statistics.py create mode 100644 src/openforms/forms/migrations/0104_formstatistics.py create mode 100644 src/openforms/forms/models/form_statistics.py create mode 100644 src/openforms/forms/tests/test_statistics.py diff --git a/src/openforms/fixtures/default_admin_index.json b/src/openforms/fixtures/default_admin_index.json index 15914c9ef3..10f753d2af 100644 --- a/src/openforms/fixtures/default_admin_index.json +++ b/src/openforms/fixtures/default_admin_index.json @@ -67,6 +67,10 @@ [ "forms", "form" + ], + [ + "forms", + "formstatistics" ] ] } diff --git a/src/openforms/forms/admin/__init__.py b/src/openforms/forms/admin/__init__.py index ff93fe942b..4eb4e5844f 100644 --- a/src/openforms/forms/admin/__init__.py +++ b/src/openforms/forms/admin/__init__.py @@ -2,6 +2,7 @@ from . import form # noqa from . import form_definition # noqa from . import form_logic # noqa +from . import form_statistics # noqa from . import form_step # noqa from . import form_variable # noqa from . import form_version # noqa diff --git a/src/openforms/forms/admin/form_statistics.py b/src/openforms/forms/admin/form_statistics.py new file mode 100644 index 0000000000..eccafaa3ac --- /dev/null +++ b/src/openforms/forms/admin/form_statistics.py @@ -0,0 +1,33 @@ +from django.contrib import admin + +from ..models import FormStatistics + + +@admin.register(FormStatistics) +class FormStatisticsAdmin(admin.ModelAdmin): + list_display = ( + "form_name", + "submission_count", + "first_submission", + "last_submission", + ) + fields = ( + "form", + "form_name", + "submission_count", + "first_submission", + "last_submission", + ) + + search_fields = ("form_name", "form") + date_hierarchy = "last_submission" + list_filter = ("first_submission", "last_submission", "submission_count") + + def has_add_permission(self, request): + return False + + def has_delete_permission(self, request, obj=None): + return False + + def has_change_permission(self, request, obj=None): + return False diff --git a/src/openforms/forms/migrations/0104_formstatistics.py b/src/openforms/forms/migrations/0104_formstatistics.py new file mode 100644 index 0000000000..4cee67f742 --- /dev/null +++ b/src/openforms/forms/migrations/0104_formstatistics.py @@ -0,0 +1,72 @@ +# Generated by Django 3.2.23 on 2024-01-19 11:42 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("forms", "0103_fix_component_problems"), + ] + + operations = [ + migrations.CreateModel( + name="FormStatistics", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "form_name", + models.CharField( + help_text="The name of the submitted form. This is saved separately in case of form deletion.", + max_length=150, + verbose_name="form name", + ), + ), + ( + "submission_count", + models.PositiveIntegerField( + default=0, + help_text="The number of the submitted forms.", + verbose_name="Submission count", + ), + ), + ( + "first_submission", + models.DateTimeField( + auto_now_add=True, + help_text="Date and time of the first submitted form.", + verbose_name="first submission", + ), + ), + ( + "last_submission", + models.DateTimeField( + help_text="Date and time of the last submitted form.", + verbose_name="last submission", + ), + ), + ( + "form", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="forms.form", + verbose_name="form", + ), + ), + ], + options={ + "verbose_name": "Form statistics", + "verbose_name_plural": "Form statistics", + }, + ), + ] diff --git a/src/openforms/forms/models/__init__.py b/src/openforms/forms/models/__init__.py index 98196efe8f..fe1ebab06f 100644 --- a/src/openforms/forms/models/__init__.py +++ b/src/openforms/forms/models/__init__.py @@ -2,6 +2,7 @@ from .form import Form, FormsExport from .form_definition import FormDefinition from .form_registration_backend import FormRegistrationBackend +from .form_statistics import FormStatistics from .form_step import FormStep from .form_variable import FormVariable from .form_version import FormVersion @@ -16,6 +17,7 @@ "FormVersion", "FormLogic", "FormPriceLogic", + "FormStatistics", "FormVariable", "Category", "FormRegistrationBackend", diff --git a/src/openforms/forms/models/form_statistics.py b/src/openforms/forms/models/form_statistics.py new file mode 100644 index 0000000000..8d9e28ee77 --- /dev/null +++ b/src/openforms/forms/models/form_statistics.py @@ -0,0 +1,40 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class FormStatistics(models.Model): + form = models.ForeignKey( + verbose_name=_("form"), + to="forms.Form", + null=True, + blank=True, + on_delete=models.SET_NULL, + ) + form_name = models.CharField( + verbose_name=_("form name"), + max_length=150, + help_text=_( + "The name of the submitted form. This is saved separately in case of form deletion." + ), + ) + submission_count = models.PositiveIntegerField( + verbose_name=_("Submission count"), + default=0, + help_text=_("The number of the submitted forms."), + ) + first_submission = models.DateTimeField( + verbose_name=_("first submission"), + help_text=_("Date and time of the first submitted form."), + auto_now_add=True, + ) + last_submission = models.DateTimeField( + verbose_name=_("last submission"), + help_text=_("Date and time of the last submitted form."), + ) + + class Meta: + verbose_name = _("Form statistics") + verbose_name_plural = _("Form statistics") + + def __str__(self): + return f"{self.form_name} last submitted on {self.last_submission}" diff --git a/src/openforms/forms/tests/test_statistics.py b/src/openforms/forms/tests/test_statistics.py new file mode 100644 index 0000000000..4f46c1c00c --- /dev/null +++ b/src/openforms/forms/tests/test_statistics.py @@ -0,0 +1,125 @@ +import datetime +from unittest.mock import patch + +from freezegun import freeze_time +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APITestCase + +from openforms.forms.tests.factories import FormFactory +from openforms.submissions.constants import PostSubmissionEvents +from openforms.submissions.tasks import retry_processing_submissions +from openforms.submissions.tests.factories import SubmissionFactory +from openforms.submissions.tests.mixins import SubmissionsMixin + +from ..models.form_statistics import FormStatistics + + +class FormStatisticsTests(SubmissionsMixin, APITestCase): + @freeze_time("2020-12-11T12:00:00+00:00") + def test_form_statistics_is_created(self): + form = FormFactory.create() + submission = SubmissionFactory.create(form=form) + self._add_submission_to_session(submission) + + endpoint = reverse("api:submission-complete", kwargs={"uuid": submission.uuid}) + response = self.client.post(endpoint, {"privacy_policy_accepted": True}) + + form_statistics = FormStatistics.objects.get() + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(form_statistics.form, form) + self.assertEqual(form_statistics.form_name, form.name) + self.assertEqual(form_statistics.submission_count, 1) + self.assertEqual( + form_statistics.first_submission, + datetime.datetime(2020, 12, 11, 12, 00, 00, tzinfo=datetime.timezone.utc), + ) + self.assertEqual( + form_statistics.last_submission, + datetime.datetime(2020, 12, 11, 12, 00, 00, tzinfo=datetime.timezone.utc), + ) + + def test_form_statistics_is_updated(self): + with freeze_time("2020-12-11T12:00:00+00:00") as frozen_datetime: + form = FormFactory.create() + + # assuming that the form has been already submitted at least one time + FormStatistics.objects.create( + form=form, + form_name=form.name, + submission_count=1, + last_submission=datetime.datetime( + 2020, 12, 11, 12, 00, 00, tzinfo=datetime.timezone.utc + ), + ) + + submission = SubmissionFactory.create(form=form) + self._add_submission_to_session(submission) + + # submit the form for the second time after 1 minute + frozen_datetime.tick(delta=datetime.timedelta(minutes=1)) + + endpoint = reverse( + "api:submission-complete", kwargs={"uuid": submission.uuid} + ) + + with self.captureOnCommitCallbacks(execute=True): + response = self.client.post(endpoint, {"privacy_policy_accepted": True}) + + form_statistics = FormStatistics.objects.get() + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(form_statistics.form, form) + self.assertEqual(form_statistics.form_name, form.name) + self.assertEqual(form_statistics.submission_count, 2) + self.assertEqual( + form_statistics.first_submission, + datetime.datetime(2020, 12, 11, 12, 00, 00, tzinfo=datetime.timezone.utc), + ) + self.assertEqual( + form_statistics.last_submission, + datetime.datetime(2020, 12, 11, 12, 1, 00, tzinfo=datetime.timezone.utc), + ) + + @patch("openforms.submissions.tasks.on_post_submission_event") + def test_resend_submission_task_does_not_affect_counter(self, m): + with freeze_time("2020-12-11T12:00:00+00:00") as frozen_datetime: + form = FormFactory.create() + + FormStatistics.objects.create( + form=form, + form_name=form.name, + submission_count=1, + last_submission=datetime.datetime( + 2020, 12, 11, 12, 00, 00, tzinfo=datetime.timezone.utc + ), + ) + failed_submission = SubmissionFactory.create( + registration_failed=True, + needs_on_completion_retry=True, + completed_on=datetime.datetime( + 2020, 12, 11, 12, 00, 00, tzinfo=datetime.timezone.utc + ), + form=form, + ) + # not failed submission + SubmissionFactory.create( + needs_on_completion_retry=False, + registration_pending=False, + completed_on=datetime.datetime( + 2020, 12, 11, 12, 18, 00, tzinfo=datetime.timezone.utc + ), + form=form, + ) + + frozen_datetime.tick(delta=datetime.timedelta(minutes=10)) + retry_processing_submissions() + + form_statistics = FormStatistics.objects.get() + + self.assertEqual(m.call_count, 1) + m.assert_called_once_with(failed_submission.id, PostSubmissionEvents.on_retry) + + self.assertEqual(form_statistics.form, form) + self.assertEqual(form_statistics.submission_count, 1) diff --git a/src/openforms/submissions/api/mixins.py b/src/openforms/submissions/api/mixins.py index 2c6d3091c5..b34e398cce 100644 --- a/src/openforms/submissions/api/mixins.py +++ b/src/openforms/submissions/api/mixins.py @@ -27,11 +27,13 @@ def _complete_submission(self, submission: Submission) -> str: This encapsulates the logic of what it means to 'complete' a submission, ensuring that the relevant metadata is set and post-completion hooks trigger, - such as schedulign the processing via Celery. + such as scheduling the processing via Celery. """ # dispatch signal for modules to tap into - submission_complete.send(sender=self.__class__, request=self.request) + submission_complete.send( + sender=self.__class__, request=self.request, instance=submission + ) submission.calculate_price(save=False) submission.completed_on = timezone.now() diff --git a/src/openforms/submissions/signals.py b/src/openforms/submissions/signals.py index f0ab0d6996..c1cd713052 100644 --- a/src/openforms/submissions/signals.py +++ b/src/openforms/submissions/signals.py @@ -3,8 +3,10 @@ from django.db.models.base import ModelBase from django.db.models.signals import post_delete from django.dispatch import Signal, receiver +from django.utils import timezone -from openforms.submissions.models import SubmissionReport +from openforms.forms.models.form_statistics import FormStatistics +from openforms.submissions.models import Submission, SubmissionReport logger = logging.getLogger(__name__) @@ -57,3 +59,22 @@ def delete_submission_report_files( logger.debug("Deleting file %r", instance.content.name) instance.content.delete(save=False) + + +@receiver(submission_complete, dispatch_uid="submission.increment_form_counter") +def increment_form_counter(sender, instance: Submission, **kwargs): + submitted_form = instance.form + + try: + existing_form_statistics = FormStatistics.objects.get(form=submitted_form) + except FormStatistics.DoesNotExist: + FormStatistics.objects.create( + form=submitted_form, + form_name=submitted_form.name, + submission_count=1, + last_submission=timezone.now(), + ) + else: + existing_form_statistics.submission_count += 1 + existing_form_statistics.last_submission = timezone.now() + existing_form_statistics.save()