Update Prepop branch with changes from main

.schemas-version

@@ -1 +1 @@
-v3.55.0
+v3.58.0

Pipfile

@@ -38,7 +38,7 @@ pytest-mock = "*"
 
 [packages]
 colorama = "*"
-flask = "*"
+flask = "==2.2.2"
 flask-babel = "*"
 flask-login = "*"
 flask-wtf = "*"
@@ -72,6 +72,7 @@ google-cloud-tasks = "*"
 simplejson = "*"
 markupsafe = "*"
 pdfkit = "*"
+ordered-set = "*"
 
 [requires]
 python_version = "3.10"

app/data_models/metadata_proxy.py

@@ -49,11 +49,11 @@ class MetadataProxy:
     case_id: str
     collection_exercise_sid: str
     response_id: str
+    response_expires_at: datetime
     survey_metadata: Optional[SurveyMetadata] = None
     schema_url: Optional[str] = None
     schema_name: Optional[str] = None
     language_code: Optional[str] = None
-    response_expires_at: Optional[datetime] = None
     channel: Optional[str] = None
     region_code: Optional[str] = None
     version: Optional[AuthPayloadVersion] = None

app/data_models/progress_store.py

@@ -111,16 +111,16 @@ def section_keys(
 
     def update_section_status(
         self, section_status: str, section_id: str, list_item_id: Optional[str] = None
-    ) -> None:
+    ) -> bool:
+        updated = False
         section_key = (section_id, list_item_id)
         if section_key in self._progress:
-            self._progress[section_key].status = section_status
-            self._is_dirty = True
+            if self._progress[section_key].status != section_status:
+                updated = True
+                self._progress[section_key].status = section_status
+                self._is_dirty = True
 
-        elif (
-            section_status == CompletionStatus.INDIVIDUAL_RESPONSE_REQUESTED
-            and section_key not in self._progress
-        ):
+        elif section_status == CompletionStatus.INDIVIDUAL_RESPONSE_REQUESTED:
             self._progress[section_key] = Progress(
                 section_id=section_id,
                 list_item_id=list_item_id,
@@ -129,6 +129,8 @@ def update_section_status(
             )
             self._is_dirty = True
 
+        return updated
+
     def get_section_status(
         self, section_id: str, list_item_id: Optional[str] = None
     ) -> str:
@@ -138,8 +140,19 @@ def get_section_status(
 
         return CompletionStatus.NOT_STARTED
 
+    def get_block_status(
+        self, *, block_id: str, section_id: str, list_item_id: str | None = None
+    ) -> str:
+        section_blocks = self.get_completed_block_ids(
+            section_id=section_id, list_item_id=list_item_id
+        )
+        if block_id in section_blocks:
+            return CompletionStatus.COMPLETED
+
+        return CompletionStatus.NOT_STARTED
+
     def get_completed_block_ids(
-        self, section_id: str, list_item_id: Optional[str] = None
+        self, *, section_id: str, list_item_id: str | None = None
     ) -> list[str]:
         section_key = (section_id, list_item_id)
         if section_key in self._progress:
@@ -151,7 +164,9 @@ def add_completed_location(self, location: Location) -> None:
         section_id = location.section_id
         list_item_id = location.list_item_id
 
-        completed_block_ids = self.get_completed_block_ids(section_id, list_item_id)
+        completed_block_ids = self.get_completed_block_ids(
+            section_id=section_id, list_item_id=list_item_id
+        )
 
         if location.block_id not in completed_block_ids:
             completed_block_ids.append(location.block_id)  # type: ignore

app/data_models/questionnaire_store.py

@@ -93,12 +93,10 @@ def save(self) -> None:
         collection_exercise_sid = (
             self.collection_exercise_sid or self._metadata["collection_exercise_sid"]
         )
-        response_expires_at = self._metadata.get("response_expires_at")
+        response_expires_at = self._metadata["response_expires_at"]
         self._storage.save(
             data=data,
             collection_exercise_sid=collection_exercise_sid,
             submitted_at=self.submitted_at,
-            expires_at=parse_iso_8601_datetime(response_expires_at)
-            if response_expires_at
-            else None,
+            expires_at=parse_iso_8601_datetime(response_expires_at),
         )

app/forms/fields/decimal_field_with_separator.py

@@ -1,9 +1,8 @@
 from decimal import Decimal, InvalidOperation
 
-from babel import numbers
 from wtforms import DecimalField
 
-from app.settings import DEFAULT_LOCALE
+from app.helpers.form_helpers import sanitise_number
 
 
 class DecimalFieldWithSeparator(DecimalField):
@@ -23,8 +22,6 @@ def __init__(self, **kwargs):
     def process_formdata(self, valuelist):
         if valuelist:
             try:
-                self.data = Decimal(
-                    valuelist[0].replace(numbers.get_group_symbol(DEFAULT_LOCALE), "")
-                )
+                self.data = Decimal(sanitise_number(valuelist[0]))
             except (ValueError, TypeError, InvalidOperation):
                 pass

app/forms/fields/integer_field_with_separator.py

@@ -1,7 +1,6 @@
-from babel import numbers
 from wtforms import IntegerField
 
-from app.settings import DEFAULT_LOCALE
+from app.helpers.form_helpers import sanitise_number
 
 
 class IntegerFieldWithSeparator(IntegerField):
@@ -21,8 +20,6 @@ def __init__(self, **kwargs):
     def process_formdata(self, valuelist):
         if valuelist:
             try:
-                self.data = int(
-                    valuelist[0].replace(numbers.get_group_symbol(DEFAULT_LOCALE), "")
-                )
+                self.data = int(sanitise_number(valuelist[0]))
             except ValueError:
                 pass

app/forms/validators.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import math
 import re
 from datetime import datetime, timezone
 from decimal import Decimal, InvalidOperation
@@ -19,10 +20,14 @@
     DecimalFieldWithSeparator,
     IntegerFieldWithSeparator,
 )
-from app.jinja_filters import format_number, get_formatted_currency
+from app.helpers.form_helpers import (
+    format_message_with_title,
+    format_playback_value,
+    sanitise_mobile_number,
+    sanitise_number,
+)
 from app.questionnaire.questionnaire_store_updater import QuestionnaireStoreUpdater
 from app.questionnaire.rules.utils import parse_datetime
-from app.utilities import safe_content
 
 if TYPE_CHECKING:
     from app.forms.questionnaire_form import QuestionnaireForm  # pragma: no cover
@@ -49,15 +54,12 @@ def __call__(
         field: Union[DecimalFieldWithSeparator, IntegerFieldWithSeparator],
     ) -> None:
         try:
-            Decimal(
-                field.raw_data[0].replace(
-                    numbers.get_group_symbol(flask_babel.get_locale()), ""
-                )
-            )
+            # number is sanitised to guard against inputs like `,NaN_` etc
+            number = Decimal(sanitise_number(number=field.raw_data[0]))
         except (ValueError, TypeError, InvalidOperation, AttributeError) as exc:
             raise validators.StopValidation(self.message) from exc
 
-        if "e" in field.raw_data[0].lower():
+        if "e" in field.raw_data[0].lower() or math.isnan(number):
             raise validators.StopValidation(self.message)
 
 
@@ -126,6 +128,7 @@ def __call__(
         field: Union[DecimalFieldWithSeparator, IntegerFieldWithSeparator],
     ) -> None:
         value: Union[int, Decimal] = field.data
+
         if value is not None:
             error_message = self.validate_minimum(value) or self.validate_maximum(value)
             if error_message:
@@ -179,11 +182,7 @@ def __init__(self, max_decimals: int = 0, messages: OptionalMessage = None):
     def __call__(
         self, form: "QuestionnaireForm", field: DecimalFieldWithSeparator
     ) -> None:
-        data = (
-            field.raw_data[0]
-            .replace(numbers.get_group_symbol(flask_babel.get_locale()), "")
-            .replace(" ", "")
-        )
+        data = sanitise_number(field.raw_data[0])
         decimal_symbol = numbers.get_decimal_symbol(flask_babel.get_locale())
         if data and decimal_symbol in data:
             if self.max_decimals == 0:
@@ -450,20 +449,6 @@ def _is_valid(
         raise NotImplementedError(f"Condition '{condition}' is not implemented")
 
 
-def format_playback_value(
-    value: Union[float, Decimal], currency: Optional[str] = None
-) -> str:
-    if currency:
-        return get_formatted_currency(value, currency)
-
-    formatted_number: str = format_number(value)
-    return formatted_number
-
-
-def format_message_with_title(error_message: str, question_title: str) -> str:
-    return error_message % {"question_title": safe_content(question_title)}
-
-
 class MutuallyExclusiveCheck:
     def __init__(self, question_title: str, messages: OptionalMessage = None):
         self.messages = {**error_messages, **(messages or {})}
@@ -492,11 +477,6 @@ def __call__(
             raise validators.ValidationError(message)
 
 
-def sanitise_mobile_number(data: str) -> str:
-    data = re.sub(r"[\s.,\t\-{}\[\]()/]", "", data)
-    return re.sub(r"^(0{1,2}44|\+44|0)", "", data)
-
-
 class MobileNumberCheck:
     def __init__(self, message: OptionalMessage = None):
         self.message = message or error_messages["INVALID_MOBILE_NUMBER"]

app/helpers/form_helpers.py

@@ -0,0 +1,33 @@
+import re
+from decimal import Decimal
+
+import flask_babel
+from babel import numbers
+
+from app.jinja_filters import format_number, get_formatted_currency
+from app.utilities import safe_content
+
+
+def sanitise_number(number: str) -> str:
+    return (
+        number.replace(numbers.get_group_symbol(flask_babel.get_locale()), "")
+        .replace("_", "")
+        .replace(" ", "")
+    )
+
+
+def sanitise_mobile_number(data: str) -> str:
+    data = re.sub(r"[\s.,\t\-{}\[\]()/]", "", data)
+    return re.sub(r"^(0{1,2}44|\+44|0)", "", data)
+
+
+def format_playback_value(value: float | Decimal, currency: str | None = None) -> str:
+    if currency:
+        return get_formatted_currency(value, currency)
+
+    formatted_number: str = format_number(value)
+    return formatted_number
+
+
+def format_message_with_title(error_message: str, question_title: str) -> str:
+    return error_message % {"question_title": safe_content(question_title)}

app/jinja_filters.py

@@ -2,7 +2,7 @@
 import re
 from datetime import datetime
 from decimal import Decimal
-from typing import Any, Callable, Mapping, Optional, Union
+from typing import Any, Callable, Literal, Mapping, Optional, TypeAlias, Union
 
 import flask
 import flask_babel
@@ -18,6 +18,7 @@
 blueprint = flask.Blueprint("filters", __name__)
 FormType = Mapping[str, Mapping[str, Any]]
 AnswerType = Mapping[str, Any]
+UnitLengthType: TypeAlias = Literal["short", "long", "narrow"]
 
 
 def mark_safe(context: nodes.EvalContext, value: str) -> Union[Markup, str]:
@@ -69,7 +70,9 @@ def format_percentage(value: Union[int, float, Decimal]) -> str:
 
 
 def format_unit(
-    unit: str, value: Union[int, float, Decimal], length: str = "short"
+    unit: str,
+    value: int | float | Decimal,
+    length: UnitLengthType = "short",
 ) -> str:
     formatted_unit: str = units.format_unit(
         value=value,
@@ -80,7 +83,7 @@ def format_unit(
     return formatted_unit
 
 
-def format_unit_input_label(unit: str, unit_length: str = "short") -> str:
+def format_unit_input_label(unit: str, unit_length: UnitLengthType = "short") -> str:
     """
     This function is used to only get the unit of measurement text. If the unit_length
     is long then only the plural form of the word is returned (e.g., Hours, Years, etc).
@@ -97,8 +100,9 @@ def format_unit_input_label(unit: str, unit_length: str = "short") -> str:
             locale=flask_babel.get_locale(),
         ).replace("2 ", "")
     else:
+        # Type ignore: We pass an empty string  as the value so that we just return the unit label
         unit_label = units.format_unit(
-            value="",
+            value="",  # type: ignore
             measurement_unit=unit,
             length=unit_length,
             locale=flask_babel.get_locale(),
@@ -183,7 +187,10 @@ def get_format_date_range(start_date: Markup, end_date: Markup) -> Markup:
 
 @blueprint.app_context_processor
 def format_unit_processor() -> (
-    dict[str, Callable[[str, Union[int, Decimal], str], str]]
+    dict[
+        str,
+        Callable[[str, int | float | Decimal, UnitLengthType], str],
+    ]
 ):
     return {"format_unit": format_unit}
 

app/publisher/publisher.py

@@ -2,6 +2,7 @@
 
 import google.auth
 from google.cloud.pubsub import PublisherClient
+from google.cloud.pubsub_v1 import publisher
 from google.cloud.pubsub_v1.futures import Future
 from structlog import get_logger
 
@@ -21,7 +22,7 @@ def __init__(self):
         self._client = PublisherClient()
         _, self._project_id = google.auth.default()
 
-    def _publish(self, topic_id, message):
+    def _publish(self, topic_id, message) -> "publisher.futures.Future":
         logger.info("publishing message", topic_id=topic_id)
         # pylint: disable=no-member
         topic_path = self._client.topic_path(self._project_id, topic_id)

app/questionnaire/dependencies.py

@@ -2,6 +2,7 @@
 
 from typing import TYPE_CHECKING, Mapping, Sequence
 
+from ordered_set import OrderedSet
 from werkzeug.datastructures import MultiDict
 
 from app.data_models import ProgressStore
@@ -30,7 +31,7 @@ def get_block_ids_for_calculated_summary_dependencies(
     ]
 
     if block_id := location.block_id:
-        dependents = dependent_sections[block_id]
+        dependents = OrderedSet(dependent_sections[block_id])
     else:
         dependents = get_flattened_mapping_values(dependent_sections)