Update Prepop branch with changes from main (#1126)

* Make response expiry date mandatory (#1104)

* Bind additional contexts to flush requests (#1108)

* Schemas v3.56.0 (#1110)

* Schemas v3.57.0 (#1113)

* Schemas v3.57.1 (#1115)

* Schemas v3.58.0 (#1116)

* Implement "progress" value source (#1044)

Co-authored-by: Rhys Berrow <[email protected]

* Fix handling of invalid values in form numerical inputs  (#1111)

* Update Chromedriver to version 113 (#1118)

* update chromedriver

* temporarily comment out line, to be fixed separately

* Feat/Grand calculated summary (#1107)

* initial ideas

* Implementation closer to calculated summaries

* better test schemas and routing evaluation

* get routing working

* dont need section id

* make routing better, start adding tests

* start fixing test schemas

* Two further schema fixes

* Fix remaining schema issue

* Add integration test

* Add functional test

* Functional tests for routing and answers not on the path

* Changing an answer updates GCS progress

* Add schema for overlapping answers

* Functionally test skip-question in grand calculated summary

* Add tests for GCS routing

* Fix errors caused by merge

* type hints, comments, remove some duplication

* fix formatting

* Update translation templates

* remove accidental update

* Fix some type hints

* Use validator branch and update test description

* refactor GCS answer format

* Fix lint and test errors

* Address PR comments

* improve context and coverage of test schemas

* Fix invalid schema and add invalid routing test

* Fix line length in test

* test schema linting

* Fix routing bug

* Make schema easier to follow

* Linting

* partial fix to routing

* Add addtional routing test case

* Remove arg from new test case

* Routing to next incomplete block for summaries

* linting error

* remove arg from router tests

* PR comments

* PR Comment

* Amend comment and improve dependencies function

* Handle merge errors

* fix gcs routing within same section

* Add test for GCS with progress value source

* Remove backtick

* update chromedriver

* Revert "update chromedriver"

This reverts commit a2e3698.

* Fix schema labels

* Revert validator branch to latest

---------

Co-authored-by: petechd <[email protected]>
Co-authored-by: Mebin Abraham <[email protected]>
Co-authored-by: Guilhem <[email protected]>

app/jinja_filters.py

@@ -12,6 +12,7 @@
 from markupsafe import Markup, escape
 from wtforms import SelectFieldBase
 
+from app.questionnaire.questionnaire_schema import is_summary_with_calculation
 from app.questionnaire.rules.utils import parse_datetime
 from app.settings import MAX_NUMBER
 
@@ -469,7 +470,7 @@ def __init__(  # noqa: C901, R0912 pylint: disable=too-complex, too-many-branche
             (
                 multiple_answers
                 or answer_type == "relationship"
-                or summary_type == "CalculatedSummary"
+                or is_summary_with_calculation(summary_type)
             )
             and "label" in answer
             and answer["label"]
@@ -553,7 +554,7 @@ def __init__(
 
         multiple_answers = len(question["answers"]) > 1
 
-        if summary_type == "CalculatedSummary" and not answers_are_editable:
+        if is_summary_with_calculation(summary_type) and not answers_are_editable:
             self.total = True
 
         for answer in question["answers"]:
@@ -598,6 +599,17 @@ def map_summary_item_config(
                     edit_link_aria_label,
                 )
             )
+        elif block.get("calculated_summary"):
+            rows.append(
+                SummaryRow(
+                    block["calculated_summary"],
+                    summary_type,
+                    answers_are_editable,
+                    no_answer_provided,
+                    edit_link_text,
+                    edit_link_aria_label,
+                )
+            )
         else:
             list_collector_rows = map_list_collector_config(
                 list_items=block["list"]["list_items"],
@@ -613,7 +625,7 @@ def map_summary_item_config(
 
             rows.extend(list_collector_rows)
 
-    if summary_type == "CalculatedSummary":
+    if is_summary_with_calculation(summary_type):
         rows.append(SummaryRow(calculated_question, summary_type, False, "", "", ""))
 
     return rows

app/questionnaire/questionnaire_schema.py

@@ -326,11 +326,8 @@ def _get_answers_by_id(self) -> dict[str, list[ImmutableDict]]:
 
     def _populate_answer_dependencies(self) -> None:
         for block in self.get_blocks():
-            if block["type"] == "CalculatedSummary":
-                answer_ids_for_block = get_calculated_summary_answer_ids(block)
-                self._update_answer_dependencies_for_calculated_summary(
-                    answer_ids_for_block, block["id"]
-                )
+            if block["type"] in {"CalculatedSummary", "GrandCalculatedSummary"}:
+                self._update_answer_dependencies_for_summary(block)
                 continue
 
             for question in self.get_all_questions_for_block(block):
@@ -363,14 +360,44 @@ def _populate_answer_dependencies(self) -> None:
                                 option["detail_answer"], block_id=block["id"]
                             )
 
-    def _update_answer_dependencies_for_calculated_summary(
-        self, calculated_summary_answer_ids: Iterable[str], block_id: str
+    def _update_answer_dependencies_for_summary(self, block: ImmutableDict) -> None:
+        if block["type"] == "CalculatedSummary":
+            self._update_answer_dependencies_for_calculated_summary_dependency(
+                calculated_summary_block=block, dependent_block=block
+            )
+        elif block["type"] == "GrandCalculatedSummary":
+            self._update_answer_dependencies_for_grand_calculated_summary(block)
+
+    def _update_answer_dependencies_for_calculated_summary_dependency(
+        self, *, calculated_summary_block: ImmutableDict, dependent_block: ImmutableDict
     ) -> None:
+        """
+        update all calculated summary answers to be dependencies of the dependent block
+        """
+        calculated_summary_answer_ids = get_calculated_summary_answer_ids(
+            calculated_summary_block
+        )
         for answer_id in calculated_summary_answer_ids:
             self._answer_dependencies_map[answer_id] |= {
-                self._get_answer_dependent_for_block_id(block_id=block_id)
+                self._get_answer_dependent_for_block_id(block_id=dependent_block["id"])
             }
 
+    def _update_answer_dependencies_for_grand_calculated_summary(
+        self, grand_calculated_summary_block: ImmutableDict
+    ) -> None:
+        grand_calculated_summary_calculated_summary_ids = (
+            get_calculation_block_ids_for_grand_calculated_summary(
+                grand_calculated_summary_block
+            )
+        )
+        for calculated_summary_id in grand_calculated_summary_calculated_summary_ids:
+            # Type ignore: safe to assume block exists
+            calculated_summary_block: ImmutableDict = self.get_block(calculated_summary_id)  # type: ignore
+            self._update_answer_dependencies_for_calculated_summary_dependency(
+                calculated_summary_block=calculated_summary_block,
+                dependent_block=grand_calculated_summary_block,
+            )
+
     def _update_answer_dependencies_for_calculations(
         self, calculations: tuple[ImmutableDict, ...], *, block_id: str
     ) -> None:
@@ -408,7 +435,7 @@ def _update_answer_dependencies_for_answer(
 
     def _update_answer_dependencies_for_dynamic_options(
         self,
-        dynamic_options_values: Mapping[str, Mapping],
+        dynamic_options_values: Mapping,
         *,
         block_id: str,
         answer_id: str,
@@ -806,6 +833,23 @@ def get_first_answer_id_for_block(self, block_id: str) -> str:
         answer_ids = self.get_answer_ids_for_block(block_id)
         return answer_ids[0]
 
+    def get_answer_format_for_calculated_summary(
+        self, calculated_summary_block_id: str
+    ) -> dict:
+        """
+        Given a calculated summary block id, find the format of the total by using the first answer
+        """
+        # Type ignore: the block will exist for any valid calculated summary id
+        calculated_summary_block: ImmutableDict = self.get_block(calculated_summary_block_id)  # type: ignore
+        first_answer_id = get_calculated_summary_answer_ids(calculated_summary_block)[0]
+        first_answer = self.get_answers_by_answer_id(first_answer_id)[0]
+        return {
+            "type": first_answer["type"].lower(),
+            "unit": first_answer.get("unit"),
+            "unit_length": first_answer.get("unit_length"),
+            "currency": first_answer.get("currency"),
+        }
+
     def get_answer_ids_for_block(self, block_id: str) -> list[str]:
         block = self.get_block(block_id)
 
@@ -1205,23 +1249,44 @@ def get_item_anchor(self, section_id: str, list_name: str) -> str | None:
                     return f"#{str(item['item_anchor_answer_id'])}"
 
 
+def is_summary_with_calculation(summary_type: str) -> bool:
+    return summary_type in {"GrandCalculatedSummary", "CalculatedSummary"}
+
+
 def get_sources_for_type_from_data(
     *,
     source_type: str,
     data: MultiDict | Mapping | Sequence,
-    ignore_keys: list,
-) -> list | None:
+    ignore_keys: list | None = None,
+) -> list:
     sources = get_mappings_with_key("source", data, ignore_keys=ignore_keys)
 
     return [source for source in sources if source["source"] == source_type]
 
 
+def get_identifiers_from_calculation_block(
+    *, calculation_block: Mapping, source_type: str
+) -> list[str]:
+    values = get_sources_for_type_from_data(
+        source_type=source_type, data=calculation_block["calculation"]["operation"]
+    )
+
+    return [value["identifier"] for value in values]
+
+
 def get_calculated_summary_answer_ids(calculated_summary_block: Mapping) -> list[str]:
     if calculated_summary_block["calculation"].get("answers_to_calculate"):
-        return calculated_summary_block["calculation"]["answers_to_calculate"]  # type: ignore
+        return list(calculated_summary_block["calculation"]["answers_to_calculate"])
 
-    values = get_mappings_with_key(
-        "source", calculated_summary_block["calculation"]["operation"]
+    return get_identifiers_from_calculation_block(
+        calculation_block=calculated_summary_block, source_type="answers"
     )
 
-    return [value["identifier"] for value in values if value["source"] == "answers"]
+
+def get_calculation_block_ids_for_grand_calculated_summary(
+    grand_calculated_summary_block: Mapping,
+) -> list[str]:
+    return get_identifiers_from_calculation_block(
+        calculation_block=grand_calculated_summary_block,
+        source_type="calculated_summary",
+    )