[#3783] Added statistics view for the submitted forms to the admin

The submissions exist in the db but are cleaned periodically. We added a
model which keeps track of the submitted forms with a counter. We keep
the form name in case the form is deleted.

src/openforms/fixtures/default_admin_index.json

@@ -67,6 +67,10 @@
             [
                 "forms",
                 "form"
+            ],
+            [
+                "forms",
+                "formstatistics"
             ]
         ]
     }

src/openforms/forms/admin/__init__.py

@@ -2,6 +2,7 @@
 from . import form  # noqa
 from . import form_definition  # noqa
 from . import form_logic  # noqa
+from . import form_statistics  # noqa
 from . import form_step  # noqa
 from . import form_variable  # noqa
 from . import form_version  # noqa

src/openforms/forms/admin/form_statistics.py

@@ -0,0 +1,33 @@
+from django.contrib import admin
+
+from ..models import FormStatistics
+
+
+@admin.register(FormStatistics)
+class FormStatisticsAdmin(admin.ModelAdmin):
+    list_display = (
+        "form_name",
+        "submission_count",
+        "first_submission",
+        "last_submission",
+    )
+    fields = (
+        "form",
+        "form_name",
+        "submission_count",
+        "first_submission",
+        "last_submission",
+    )
+
+    search_fields = ("form_name", "form")
+    date_hierarchy = "last_submission"
+    list_filter = ("first_submission", "last_submission", "submission_count")
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False

src/openforms/forms/migrations/0104_formstatistics.py

@@ -0,0 +1,72 @@
+# Generated by Django 3.2.23 on 2024-01-19 11:42
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("forms", "0103_fix_component_problems"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="FormStatistics",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "form_name",
+                    models.CharField(
+                        help_text="The name of the submitted form. This is saved separately in case of form deletion.",
+                        max_length=150,
+                        verbose_name="form name",
+                    ),
+                ),
+                (
+                    "submission_count",
+                    models.PositiveIntegerField(
+                        default=0,
+                        help_text="The number of the submitted forms.",
+                        verbose_name="Submission count",
+                    ),
+                ),
+                (
+                    "first_submission",
+                    models.DateTimeField(
+                        auto_now_add=True,
+                        help_text="Date and time of the first submitted form.",
+                        verbose_name="first submission",
+                    ),
+                ),
+                (
+                    "last_submission",
+                    models.DateTimeField(
+                        help_text="Date and time of the last submitted form.",
+                        verbose_name="last submission",
+                    ),
+                ),
+                (
+                    "form",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to="forms.form",
+                        verbose_name="form",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Form statistics",
+                "verbose_name_plural": "Form statistics",
+            },
+        ),
+    ]

src/openforms/forms/models/__init__.py

@@ -2,6 +2,7 @@
 from .form import Form, FormsExport
 from .form_definition import FormDefinition
 from .form_registration_backend import FormRegistrationBackend
+from .form_statistics import FormStatistics
 from .form_step import FormStep
 from .form_variable import FormVariable
 from .form_version import FormVersion
@@ -16,6 +17,7 @@
     "FormVersion",
     "FormLogic",
     "FormPriceLogic",
+    "FormStatistics",
     "FormVariable",
     "Category",
     "FormRegistrationBackend",

src/openforms/forms/models/form_statistics.py

@@ -0,0 +1,40 @@
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+
+class FormStatistics(models.Model):
+    form = models.ForeignKey(
+        verbose_name=_("form"),
+        to="forms.Form",
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+    )
+    form_name = models.CharField(
+        verbose_name=_("form name"),
+        max_length=150,
+        help_text=_(
+            "The name of the submitted form. This is saved separately in case of form deletion."
+        ),
+    )
+    submission_count = models.PositiveIntegerField(
+        verbose_name=_("Submission count"),
+        default=0,
+        help_text=_("The number of the submitted forms."),
+    )
+    first_submission = models.DateTimeField(
+        verbose_name=_("first submission"),
+        help_text=_("Date and time of the first submitted form."),
+        auto_now_add=True,
+    )
+    last_submission = models.DateTimeField(
+        verbose_name=_("last submission"),
+        help_text=_("Date and time of the last submitted form."),
+    )
+
+    class Meta:
+        verbose_name = _("Form statistics")
+        verbose_name_plural = _("Form statistics")
+
+    def __str__(self):
+        return f"{self.form_name} last submitted on {self.last_submission}"

src/openforms/forms/tests/test_statistics.py

@@ -0,0 +1,125 @@
+import datetime
+from unittest.mock import patch
+
+from freezegun import freeze_time
+from rest_framework import status
+from rest_framework.reverse import reverse
+from rest_framework.test import APITestCase
+
+from openforms.forms.tests.factories import FormFactory
+from openforms.submissions.constants import PostSubmissionEvents
+from openforms.submissions.tasks import retry_processing_submissions
+from openforms.submissions.tests.factories import SubmissionFactory
+from openforms.submissions.tests.mixins import SubmissionsMixin
+
+from ..models.form_statistics import FormStatistics
+
+
+class FormStatisticsTests(SubmissionsMixin, APITestCase):
+    @freeze_time("2020-12-11T12:00:00+00:00")
+    def test_form_statistics_is_created(self):
+        form = FormFactory.create()
+        submission = SubmissionFactory.create(form=form)
+        self._add_submission_to_session(submission)
+
+        endpoint = reverse("api:submission-complete", kwargs={"uuid": submission.uuid})
+        response = self.client.post(endpoint, {"privacy_policy_accepted": True})
+
+        form_statistics = FormStatistics.objects.get()
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(form_statistics.form, form)
+        self.assertEqual(form_statistics.form_name, form.name)
+        self.assertEqual(form_statistics.submission_count, 1)
+        self.assertEqual(
+            form_statistics.first_submission,
+            datetime.datetime(2020, 12, 11, 12, 00, 00, tzinfo=datetime.timezone.utc),
+        )
+        self.assertEqual(
+            form_statistics.last_submission,
+            datetime.datetime(2020, 12, 11, 12, 00, 00, tzinfo=datetime.timezone.utc),
+        )
+
+    def test_form_statistics_is_updated(self):
+        with freeze_time("2020-12-11T12:00:00+00:00") as frozen_datetime:
+            form = FormFactory.create()
+
+            # assuming that the form has been already submitted at least one time
+            FormStatistics.objects.create(
+                form=form,
+                form_name=form.name,
+                submission_count=1,
+                last_submission=datetime.datetime(
+                    2020, 12, 11, 12, 00, 00, tzinfo=datetime.timezone.utc
+                ),
+            )
+
+            submission = SubmissionFactory.create(form=form)
+            self._add_submission_to_session(submission)
+
+            # submit the form for the second time after 1 minute
+            frozen_datetime.tick(delta=datetime.timedelta(minutes=1))
+
+            endpoint = reverse(
+                "api:submission-complete", kwargs={"uuid": submission.uuid}
+            )
+
+            with self.captureOnCommitCallbacks(execute=True):
+                response = self.client.post(endpoint, {"privacy_policy_accepted": True})
+
+            form_statistics = FormStatistics.objects.get()
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(form_statistics.form, form)
+        self.assertEqual(form_statistics.form_name, form.name)
+        self.assertEqual(form_statistics.submission_count, 2)
+        self.assertEqual(
+            form_statistics.first_submission,
+            datetime.datetime(2020, 12, 11, 12, 00, 00, tzinfo=datetime.timezone.utc),
+        )
+        self.assertEqual(
+            form_statistics.last_submission,
+            datetime.datetime(2020, 12, 11, 12, 1, 00, tzinfo=datetime.timezone.utc),
+        )
+
+    @patch("openforms.submissions.tasks.on_post_submission_event")
+    def test_resend_submission_task_does_not_affect_counter(self, m):
+        with freeze_time("2020-12-11T12:00:00+00:00") as frozen_datetime:
+            form = FormFactory.create()
+
+            FormStatistics.objects.create(
+                form=form,
+                form_name=form.name,
+                submission_count=1,
+                last_submission=datetime.datetime(
+                    2020, 12, 11, 12, 00, 00, tzinfo=datetime.timezone.utc
+                ),
+            )
+            failed_submission = SubmissionFactory.create(
+                registration_failed=True,
+                needs_on_completion_retry=True,
+                completed_on=datetime.datetime(
+                    2020, 12, 11, 12, 00, 00, tzinfo=datetime.timezone.utc
+                ),
+                form=form,
+            )
+            # not failed submission
+            SubmissionFactory.create(
+                needs_on_completion_retry=False,
+                registration_pending=False,
+                completed_on=datetime.datetime(
+                    2020, 12, 11, 12, 18, 00, tzinfo=datetime.timezone.utc
+                ),
+                form=form,
+            )
+
+            frozen_datetime.tick(delta=datetime.timedelta(minutes=10))
+            retry_processing_submissions()
+
+            form_statistics = FormStatistics.objects.get()
+
+        self.assertEqual(m.call_count, 1)
+        m.assert_called_once_with(failed_submission.id, PostSubmissionEvents.on_retry)
+
+        self.assertEqual(form_statistics.form, form)
+        self.assertEqual(form_statistics.submission_count, 1)

src/openforms/submissions/api/mixins.py

@@ -27,11 +27,13 @@ def _complete_submission(self, submission: Submission) -> str:
 
         This encapsulates the logic of what it means to 'complete' a submission,
         ensuring that the relevant metadata is set and post-completion hooks trigger,
-        such as schedulign the processing via Celery.
+        such as scheduling the processing via Celery.
         """
 
         # dispatch signal for modules to tap into
-        submission_complete.send(sender=self.__class__, request=self.request)
+        submission_complete.send(
+            sender=self.__class__, request=self.request, instance=submission
+        )
 
         submission.calculate_price(save=False)
         submission.completed_on = timezone.now()

src/openforms/submissions/signals.py

@@ -3,8 +3,10 @@
 from django.db.models.base import ModelBase
 from django.db.models.signals import post_delete
 from django.dispatch import Signal, receiver
+from django.utils import timezone
 
-from openforms.submissions.models import SubmissionReport
+from openforms.forms.models.form_statistics import FormStatistics
+from openforms.submissions.models import Submission, SubmissionReport
 
 logger = logging.getLogger(__name__)
 
@@ -57,3 +59,22 @@ def delete_submission_report_files(
     logger.debug("Deleting file %r", instance.content.name)
 
     instance.content.delete(save=False)
+
+
+@receiver(submission_complete, dispatch_uid="submission.increment_form_counter")
+def increment_form_counter(sender, instance: Submission, **kwargs):
+    submitted_form = instance.form
+
+    try:
+        existing_form_statistics = FormStatistics.objects.get(form=submitted_form)
+    except FormStatistics.DoesNotExist:
+        FormStatistics.objects.create(
+            form=submitted_form,
+            form_name=submitted_form.name,
+            submission_count=1,
+            last_submission=timezone.now(),
+        )
+    else:
+        existing_form_statistics.submission_count += 1
+        existing_form_statistics.last_submission = timezone.now()
+        existing_form_statistics.save()