From 35a65b9967cf95581977dbe6399247880c578dd7 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 26 Nov 2021 17:33:33 +0200 Subject: [PATCH] Add new poll results models --- ureport/backend/tests/test_rapidpro.py | 4 +- ureport/sql/stats_0028.sql | 61 ++++++ .../migrations/0027_auto_20211202_1422.py | 173 +++++++++++++++ .../migrations/0028_install_cea_triggers.py | 14 ++ ureport/stats/models.py | 80 +++++++ ureport/stats/tests.py | 198 ++++++++++++++++++ 6 files changed, 528 insertions(+), 2 deletions(-) create mode 100644 ureport/sql/stats_0028.sql create mode 100644 ureport/stats/migrations/0027_auto_20211202_1422.py create mode 100644 ureport/stats/migrations/0028_install_cea_triggers.py diff --git a/ureport/backend/tests/test_rapidpro.py b/ureport/backend/tests/test_rapidpro.py index 4e760beed..2e75aeacc 100644 --- a/ureport/backend/tests/test_rapidpro.py +++ b/ureport/backend/tests/test_rapidpro.py @@ -1230,7 +1230,7 @@ def release_boundary(boundary): ] ) - with self.assertNumQueries(7): + with self.assertNumQueries(9): boundaries_results = self.backend.pull_boundaries(self.nigeria) self.assertEqual( @@ -1276,7 +1276,7 @@ def release_boundary(boundary): ] ) - with self.assertNumQueries(13): + with self.assertNumQueries(17): boundaries_results = self.backend.pull_boundaries(self.nigeria) self.assertEqual( diff --git a/ureport/sql/stats_0028.sql b/ureport/sql/stats_0028.sql new file mode 100644 index 000000000..9f9b3a223 --- /dev/null +++ b/ureport/sql/stats_0028.sql @@ -0,0 +1,61 @@ +----------------------------------------------------------------------------- +-- Insert missing PollContactResult's ContactEngagementActivity rows +----------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION + ureport_insert_missing_contact_engagement_activities(_poll_contact_result stats_pollcontactresult) +RETURNS VOID AS $$ +BEGIN + INSERT INTO stats_contactengagementactivity(contact, date, org_id) WITH month_days(missing_month) AS ( + SELECT generate_series(date_trunc('month', _poll_contact_result.date)::timestamp,(date_trunc('month', _poll_contact_result.date)::timestamp+ interval '11 months')::date,interval '1 month')::date + ), curr_activity AS ( + SELECT * FROM stats_contactengagementactivity WHERE org_id = _poll_contact_result.org_id and contact = _poll_contact_result.contact + ) SELECT _poll_contact_result.contact, missing_month::date, _poll_contact_result.org_id FROM month_days LEFT JOIN stats_contactengagementactivity ON stats_contactengagementactivity.date = month_days.missing_month AND stats_contactengagementactivity.contact = _poll_contact_result.contact AND org_id = _poll_contact_result.org_id + WHERE stats_contactengagementactivity.date IS NULL; + UPDATE stats_contactengagementactivity SET age_segment_id = _poll_contact_result.age_segment_id, gender_segment_id = _poll_contact_result.gender_segment_id, location_id = _poll_contact_result.location_id, scheme_segment_id = _poll_contact_result.scheme_segment_id, used = TRUE WHERE org_id = _poll_contact_result.org_id and contact = _poll_contact_result.contact and date > date_trunc('month', CURRENT_DATE) - INTERVAL '1 year'; +END; +$$ LANGUAGE plpgsql; + +----------------------------------------------------------------------------- +-- Generate ContactEngagementActivity rows for latest PollContactResult +----------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION generate_contact_engagement_activities_for_latest_poll_contact_result(_poll_contact_result stats_pollcontactresult) +RETURNS VOID AS $$ +BEGIN + -- Count only if we have an org and a flow and a flow_result + IF _poll_contact_result.org_id IS NOT NULL AND _poll_contact_result.flow IS NOT NULL AND _poll_contact_result.flow_result_id IS NOT NULL AND _poll_contact_result.flow_result_category_id IS NOT NULL THEN + PERFORM ureport_insert_missing_contact_engagement_activities(_poll_contact_result); + END IF; +END; +$$ LANGUAGE plpgsql; + +----------------------------------------------------------------------------- +-- Updates our results counters +----------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION ureport_update_contact_engagement_activities() RETURNS TRIGGER AS $$ +BEGIN + -- PollContactResult row being created, increment counters for PollContactResult NEW + IF TG_OP = 'INSERT' THEN + PERFORM generate_contact_engagement_activities_for_latest_poll_contact_result(NEW); + ELSIF TG_OP = 'UPDATE' THEN + PERFORM generate_contact_engagement_activities_for_latest_poll_contact_result(NEW); + -- PollContactResult row is being deleted + ELSIF TG_OP = 'TRUNCATE' THEN + -- Clear all ContactEngagementActivity rows + TRUNCATE stats_contactengagementactivity; + END IF; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + + +-- Install trigger for INSERT, UPDATE, AND DELETE on stats_pollcontactresult +DROP TRIGGER IF EXISTS ureport_when_poll_contact_result_contact_engagement_activities on stats_pollcontactresult; +CREATE TRIGGER ureport_when_poll_contact_result_contact_engagement_activities + AFTER INSERT OR DELETE OR UPDATE ON stats_pollcontactresult + FOR EACH ROW EXECUTE PROCEDURE ureport_update_contact_engagement_activities(); + +-- Install trigger for TRUNCATE on stats_pollcontactresult +DROP TRIGGER IF EXISTS ureport_when_poll_contact_results_truncate_then_update_contact_engagement_activities ON stats_pollcontactresult; +CREATE TRIGGER ureport_when_poll_contact_results_truncate_then_update_contact_engagement_activities + AFTER TRUNCATE ON stats_pollcontactresult + EXECUTE PROCEDURE ureport_update_contact_engagement_activities(); \ No newline at end of file diff --git a/ureport/stats/migrations/0027_auto_20211202_1422.py b/ureport/stats/migrations/0027_auto_20211202_1422.py new file mode 100644 index 000000000..ddb7faf84 --- /dev/null +++ b/ureport/stats/migrations/0027_auto_20211202_1422.py @@ -0,0 +1,173 @@ +# Generated by Django 3.2.8 on 2021-12-02 14:22 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("orgs", "0029_auto_20211025_1504"), + ("flows", "0001_initial"), + ("locations", "0006_boundary_backend"), + ("stats", "0026_populate_flow_result_word_clouds"), + ] + + operations = [ + migrations.CreateModel( + name="PollContactResult", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("contact", models.CharField(max_length=36)), + ("flow", models.CharField(max_length=36)), + ("text", models.TextField(null=True)), + ("date", models.DateTimeField(null=True)), + ( + "age_segment", + models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to="stats.agesegment"), + ), + ( + "flow_result", + models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to="flows.flowresult"), + ), + ( + "flow_result_category", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.SET_NULL, to="flows.flowresultcategory" + ), + ), + ( + "gender_segment", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.SET_NULL, to="stats.gendersegment" + ), + ), + ( + "location", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.SET_NULL, to="locations.boundary" + ), + ), + ( + "org", + models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.PROTECT, + related_name="poll_contact_results", + to="orgs.org", + ), + ), + ( + "scheme_segment", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.SET_NULL, to="stats.schemesegment" + ), + ), + ], + ), + migrations.CreateModel( + name="ContactEngagementActivity", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("contact", models.CharField(max_length=36)), + ("date", models.DateField(help_text="The starting date for for the month")), + ("used", models.BooleanField(null=True)), + ( + "age_segment", + models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to="stats.agesegment"), + ), + ( + "gender_segment", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.SET_NULL, to="stats.gendersegment" + ), + ), + ( + "location", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.SET_NULL, to="locations.boundary" + ), + ), + ( + "org", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="contact_engagement_activities", + to="orgs.org", + ), + ), + ( + "scheme_segment", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.SET_NULL, to="stats.schemesegment" + ), + ), + ], + ), + migrations.AddIndex( + model_name="pollcontactresult", + index=models.Index(fields=["contact"], name="stats_pcr_contact"), + ), + migrations.AddIndex( + model_name="pollcontactresult", + index=models.Index(fields=["org", "flow", "contact"], name="stats_pcr_org_flow_contact"), + ), + migrations.AddIndex( + model_name="pollcontactresult", + index=models.Index(fields=["org", "flow"], name="stats_pcr_org_flow"), + ), + migrations.AddIndex( + model_name="pollcontactresult", + index=models.Index(fields=["org", "flow", "flow_result", "text"], name="stats_pcr_org_flow_result_text"), + ), + migrations.AddIndex( + model_name="contactengagementactivity", + index=models.Index(fields=["org", "contact"], name="stats_cea_org_contact"), + ), + migrations.AddIndex( + model_name="contactengagementactivity", + index=models.Index(fields=["org", "date"], name="stats_cea_org_date"), + ), + migrations.AddIndex( + model_name="contactengagementactivity", + index=models.Index( + condition=models.Q(("used", True)), fields=["org", "date", "used"], name="stats_cea_org_date_used" + ), + ), + migrations.AddIndex( + model_name="contactengagementactivity", + index=models.Index( + condition=models.Q(("location__isnull", False), ("used", True)), + fields=["org", "date", "location", "used"], + name="stats_cea_org_date_state_used", + ), + ), + migrations.AddIndex( + model_name="contactengagementactivity", + index=models.Index( + condition=models.Q(("age_segment__isnull", False), ("used", True)), + fields=["org", "date", "age_segment", "used"], + name="stats_cea_org_date_age_used", + ), + ), + migrations.AddIndex( + model_name="contactengagementactivity", + index=models.Index( + condition=models.Q(("scheme_segment__isnull", False), ("used", True)), + fields=["org", "date", "scheme_segment", "used"], + name="stats_cea_org_date_scheme_used", + ), + ), + migrations.AddIndex( + model_name="contactengagementactivity", + index=models.Index( + condition=models.Q(("gender_segment__isnull", False), ("used", True)), + fields=["org", "date", "gender_segment", "used"], + name="stats_cea_org_date_gender_used", + ), + ), + migrations.AlterUniqueTogether( + name="contactengagementactivity", + unique_together={("org", "contact", "date")}, + ), + ] diff --git a/ureport/stats/migrations/0028_install_cea_triggers.py b/ureport/stats/migrations/0028_install_cea_triggers.py new file mode 100644 index 000000000..6e8a42d03 --- /dev/null +++ b/ureport/stats/migrations/0028_install_cea_triggers.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.8 on 2021-12-02 14:23 + +from django.db import migrations + +from ureport.sql import InstallSQL + + +class Migration(migrations.Migration): + + dependencies = [ + ("stats", "0027_auto_20211202_1422"), + ] + + operations = [InstallSQL("stats_0028")] diff --git a/ureport/stats/models.py b/ureport/stats/models.py index 4eef49da6..858dd50fb 100644 --- a/ureport/stats/models.py +++ b/ureport/stats/models.py @@ -715,6 +715,86 @@ def calculate_average_response_rate(cls, org): return percentage +class PollContactResult(models.Model): + org = models.ForeignKey(Org, on_delete=models.PROTECT, related_name="poll_contact_results", db_index=False) + + contact = models.CharField(max_length=36) + + flow = models.CharField(max_length=36) + + flow_result = models.ForeignKey(FlowResult, null=True, on_delete=models.SET_NULL) + + flow_result_category = models.ForeignKey(FlowResultCategory, null=True, on_delete=models.SET_NULL) + + text = models.TextField(null=True) + + age_segment = models.ForeignKey(AgeSegment, null=True, on_delete=models.SET_NULL) + + gender_segment = models.ForeignKey(GenderSegment, null=True, on_delete=models.SET_NULL) + + scheme_segment = models.ForeignKey(SchemeSegment, null=True, on_delete=models.SET_NULL) + + location = models.ForeignKey(Boundary, null=True, on_delete=models.SET_NULL) + + date = models.DateTimeField(null=True) + + class Meta: + indexes = [ + models.Index(name="%(app_label)s_pcr_contact", fields=["contact"]), + models.Index(name="%(app_label)s_pcr_org_flow_contact", fields=["org", "flow", "contact"]), + models.Index(name="%(app_label)s_pcr_org_flow", fields=["org", "flow"]), + models.Index(name="%(app_label)s_pcr_org_flow_result_text", fields=["org", "flow", "flow_result", "text"]), + ] + + +class ContactEngagementActivity(models.Model): + org = models.ForeignKey(Org, on_delete=models.PROTECT, related_name="contact_engagement_activities") + + contact = models.CharField(max_length=36) + + age_segment = models.ForeignKey(AgeSegment, null=True, on_delete=models.SET_NULL) + + gender_segment = models.ForeignKey(GenderSegment, null=True, on_delete=models.SET_NULL) + + scheme_segment = models.ForeignKey(SchemeSegment, null=True, on_delete=models.SET_NULL) + + location = models.ForeignKey(Boundary, null=True, on_delete=models.SET_NULL) + + date = models.DateField(help_text="The starting date for for the month") + + used = models.BooleanField(null=True) + + class Meta: + indexes = [ + models.Index(name="%(app_label)s_cea_org_contact", fields=["org", "contact"]), + models.Index(name="%(app_label)s_cea_org_date", fields=["org", "date"]), + models.Index( + name="%(app_label)s_cea_org_date_used", fields=["org", "date", "used"], condition=Q(used=True) + ), + models.Index( + name="%(app_label)s_cea_org_date_state_used", + fields=["org", "date", "location", "used"], + condition=Q(location__isnull=False) & Q(used=True), + ), + models.Index( + name="%(app_label)s_cea_org_date_age_used", + fields=["org", "date", "age_segment", "used"], + condition=Q(age_segment__isnull=False) & Q(used=True), + ), + models.Index( + name="%(app_label)s_cea_org_date_scheme_used", + fields=["org", "date", "scheme_segment", "used"], + condition=Q(scheme_segment__isnull=False) & Q(used=True), + ), + models.Index( + name="%(app_label)s_cea_org_date_gender_used", + fields=["org", "date", "gender_segment", "used"], + condition=Q(gender_segment__isnull=False) & Q(used=True), + ), + ] + unique_together = ("org", "contact", "date") + + class ContactActivity(models.Model): org = models.ForeignKey(Org, on_delete=models.PROTECT, related_name="contact_activities") diff --git a/ureport/stats/tests.py b/ureport/stats/tests.py index e69de29bb..d4e73d7a2 100644 --- a/ureport/stats/tests.py +++ b/ureport/stats/tests.py @@ -0,0 +1,198 @@ +from datetime import timedelta + +from django.utils import timezone + +from dash.categories.models import Category +from ureport.flows.models import FlowResult, FlowResultCategory +from ureport.locations.models import Boundary +from ureport.stats.models import AgeSegment, ContactEngagementActivity, GenderSegment, PollContactResult, SchemeSegment +from ureport.tests import UreportTest + + +class PollResultsTest(UreportTest): + def setUp(self): + super(PollResultsTest, self).setUp() + + self.education_nigeria = Category.objects.create( + org=self.nigeria, name="Education", created_by=self.admin, modified_by=self.admin + ) + + self.poll = self.create_poll(self.nigeria, "Poll 1", "flow-uuid", self.education_nigeria, self.admin) + + self.flow_result, created = FlowResult.objects.get_or_create( + org=self.poll.org, flow_uuid=self.poll.flow_uuid, result_uuid="step-uuid", result_name="question 1" + ) + + self.flow_result_category_no, created = FlowResultCategory.objects.get_or_create( + flow_result_id=self.flow_result.id, category="No", is_active=True + ) + + self.flow_result_category_yes, created = FlowResultCategory.objects.get_or_create( + flow_result_id=self.flow_result.id, category="Yes", is_active=True + ) + + # boundaries fetched + self.country = Boundary.objects.create( + org=self.nigeria, + osm_id="R-NIGERIA", + name="Nigeria", + backend=self.rapidpro_backend, + level=Boundary.COUNTRY_LEVEL, + parent=None, + geometry='{"foo":"bar-country"}', + ) + self.state = Boundary.objects.create( + org=self.nigeria, + osm_id="R-LAGOS", + name="Lagos", + backend=self.rapidpro_backend, + level=Boundary.STATE_LEVEL, + parent=self.country, + geometry='{"foo":"bar-state"}', + ) + self.district = Boundary.objects.create( + org=self.nigeria, + osm_id="R-OYO", + name="Oyo", + backend=self.rapidpro_backend, + level=Boundary.DISTRICT_LEVEL, + parent=self.state, + geometry='{"foo":"bar-state"}', + ) + self.ward = Boundary.objects.create( + org=self.nigeria, + osm_id="R-IKEJA", + name="Ikeja", + backend=self.rapidpro_backend, + level=Boundary.WARD_LEVEL, + parent=self.district, + geometry='{"foo":"bar-state"}', + ) + + self.female_gender = GenderSegment.objects.get(gender="F") + self.male_gender = GenderSegment.objects.get(gender="M") + + self.age1 = AgeSegment.objects.all().order_by("min_age").first() + + self.tel_scheme = SchemeSegment.objects.create(scheme="tel") + + self.now = timezone.now() + self.last_week = self.now - timedelta(days=7) + self.last_month = self.now - timedelta(days=30) + + def test_contact_engagement_activity(self): + self.assertFalse(ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid")) + + PollContactResult.objects.create( + org=self.nigeria, + flow=self.poll.flow_uuid, + flow_result=self.flow_result, + date=self.now, + contact="contact-uuid", + ) + + self.assertFalse(ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid")) + + PollContactResult.objects.create( + org=self.nigeria, + flow=self.poll.flow_uuid, + flow_result=self.flow_result, + flow_result_category=self.flow_result_category_no, + date=self.now, + contact="contact-uuid", + ) + + self.assertTrue(ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid")) + self.assertEqual( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid").count(), 12 + ) + self.assertFalse( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid").exclude( + age_segment=None + ) + ) + self.assertFalse( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid").exclude( + gender_segment=None + ) + ) + self.assertFalse( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid").exclude(location=None) + ) + self.assertFalse( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid").exclude( + scheme_segment=None + ) + ) + + PollContactResult.objects.create( + org=self.nigeria, + flow=self.poll.flow_uuid, + flow_result=self.flow_result, + flow_result_category=self.flow_result_category_no, + contact="contact-uuid", + text="Nah", + date=self.now, + location=self.ward, + scheme_segment=self.tel_scheme, + ) + + self.assertTrue(ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid")) + self.assertEqual( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid").count(), 12 + ) + self.assertFalse( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid").exclude( + age_segment=None + ) + ) + self.assertFalse( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid").exclude( + gender_segment=None + ) + ) + self.assertTrue( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid").exclude(location=None) + ) + self.assertTrue( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid").exclude( + scheme_segment=None + ) + ) + + PollContactResult.objects.create( + org=self.nigeria, + flow=self.poll.flow_uuid, + flow_result=self.flow_result, + flow_result_category=self.flow_result_category_yes, + contact="contact-uuid2", + text="Yeah", + age_segment=self.age1, + gender_segment=self.male_gender, + date=self.now, + location=self.ward, + scheme_segment=self.tel_scheme, + ) + + self.assertTrue(ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid2")) + self.assertEqual( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid2").count(), 12 + ) + self.assertTrue( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid2").exclude( + age_segment=None + ) + ) + self.assertTrue( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid2").exclude( + gender_segment=None + ) + ) + self.assertTrue( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid2").exclude(location=None) + ) + self.assertTrue( + ContactEngagementActivity.objects.filter(org=self.nigeria, contact="contact-uuid2").exclude( + scheme_segment=None + ) + )