Integrate with SDS endpoint to support loading Prepop data

.development.env

@@ -27,3 +27,4 @@ CDN_ASSETS_PATH=/design-system
 ADDRESS_LOOKUP_API_URL=https://whitelodge-ai-api.census-gcp.onsdigital.uk
 COOKIE_SETTINGS_URL=#
 EQ_SUBMISSION_CONFIRMATION_BACKEND=log
+SDS_API_BASE_URL=http://localhost:5003/v1/unit_data

.functional-tests.env

@@ -28,3 +28,4 @@ ADDRESS_LOOKUP_API_URL=https://whitelodge-ai-api.census-gcp.onsdigital.uk
 COOKIE_SETTINGS_URL=#
 EQ_SUBMISSION_CONFIRMATION_BACKEND=log
 VIEW_SUBMITTED_RESPONSE_EXPIRATION_IN_SECONDS=35
+SDS_API_BASE_URL=http://localhost:5003/v1/unit_data

app/authentication/auth_payload_version.py → app/authentication/auth_payload_versions.py

@@ -3,3 +3,7 @@
 
 class AuthPayloadVersion(Enum):
     V2 = "v2"
+
+
+class SupplementaryDataSchemaVersion(Enum):
+    V1 = "v1"

app/data_models/metadata_proxy.py

@@ -7,7 +7,7 @@
 
 from werkzeug.datastructures import ImmutableDict
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.utilities.make_immutable import make_immutable
 
 

app/routes/errors.py

@@ -22,6 +22,7 @@
 from app.globals import get_metadata
 from app.helpers.language_helper import handle_language
 from app.helpers.template_helpers import get_survey_config, render_template
+from app.services.supplementary_data import SupplementaryDataRequestFailed
 from app.settings import ACCOUNT_SERVICE_BASE_URL_SOCIAL
 from app.submitter.previously_submitted_exception import PreviouslySubmittedException
 from app.submitter.submission_failed import SubmissionFailedException
@@ -189,6 +190,14 @@ def too_many_feedback_requests(
     )
 
 
+@errors_blueprint.app_errorhandler(SupplementaryDataRequestFailed)
+def supplementary_data_request_failed(
+    exception: SupplementaryDataRequestFailed,
+) -> tuple[str, int]:
+    log_exception(exception, 500)
+    return _render_error_page(500, template=500)
+
+
 @errors_blueprint.app_errorhandler(SubmissionFailedException)
 def submission_failed(
     exception: SubmissionFailedException,

app/routes/flush.py

@@ -7,7 +7,7 @@
 from sdc.crypto.key_store import KeyStore
 from structlog import contextvars, get_logger
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.authentication.user import User
 from app.authentication.user_id_generator import UserIDGenerator
 from app.data_models import QuestionnaireStore

app/routes/session.py

@@ -11,7 +11,7 @@
 from werkzeug.exceptions import Unauthorized
 from werkzeug.wrappers.response import Response
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.authentication.authenticator import decrypt_token, store_session
 from app.authentication.jti_claim_storage import JtiTokenUsed, use_jti_claim
 from app.data_models.metadata_proxy import MetadataProxy
@@ -23,6 +23,7 @@
 )
 from app.questionnaire import QuestionnaireSchema
 from app.routes.errors import _render_error_page
+from app.services.supplementary_data import get_supplementary_data
 from app.utilities.metadata_parser import validate_runner_claims
 from app.utilities.metadata_parser_v2 import (
     validate_questionnaire_claims,
@@ -128,6 +129,14 @@ def login() -> Response:
 
     cookie_session["language_code"] = metadata.language_code
 
+    # Type ignore: survey_id and either ru_ref or qid are required for schemas that use supplementary data
+    if dataset_id := metadata["sds_dataset_id"]:
+        get_supplementary_data(
+            dataset_id=dataset_id,
+            unit_id=metadata["ru_ref"] or metadata["qid"],  # type: ignore
+            survey_id=metadata["survey_id"],  # type: ignore
+        )
+
     return redirect(url_for("questionnaire.get_questionnaire"))
 
 

app/services/supplementary_data.py

@@ -0,0 +1,92 @@
+import json
+from typing import Mapping
+from urllib.parse import urlencode
+
+from flask import current_app
+from marshmallow import ValidationError
+from requests import RequestException
+from structlog import get_logger
+
+from app.utilities.request_session import get_retryable_session
+from app.utilities.supplementary_data_parser import validate_supplementary_data_v1
+
+SUPPLEMENTARY_DATA_REQUEST_BACKOFF_FACTOR = 0.2
+SUPPLEMENTARY_DATA_REQUEST_MAX_RETRIES = 2  # Totals no. of request should be 3. The initial request + SUPPLEMENTARY_DATA_REQUEST_MAX_RETRIES
+SUPPLEMENTARY_DATA_REQUEST_TIMEOUT = 3
+SUPPLEMENTARY_DATA_REQUEST_RETRY_STATUS_CODES = [
+    408,
+    429,
+    500,
+    502,
+    503,
+    504,
+]
+
+logger = get_logger()
+
+
+class SupplementaryDataRequestFailed(Exception):
+    def __str__(self) -> str:
+        return "Supplementary Data request failed"
+
+
+def get_supplementary_data(*, dataset_id: str, unit_id: str, survey_id: str) -> dict:
+    supplementary_data_url = current_app.config["SDS_API_BASE_URL"]
+
+    parameters = {"dataset_id": dataset_id, "unit_id": unit_id}
+
+    encoded_parameters = urlencode(parameters)
+    constructed_supplementary_data_url = (
+        f"{supplementary_data_url}?{encoded_parameters}"
+    )
+
+    session = get_retryable_session(
+        max_retries=SUPPLEMENTARY_DATA_REQUEST_MAX_RETRIES,
+        retry_status_codes=SUPPLEMENTARY_DATA_REQUEST_RETRY_STATUS_CODES,
+        backoff_factor=SUPPLEMENTARY_DATA_REQUEST_BACKOFF_FACTOR,
+    )
+
+    try:
+        response = session.get(
+            constructed_supplementary_data_url,
+            timeout=SUPPLEMENTARY_DATA_REQUEST_TIMEOUT,
+        )
+    except RequestException as exc:
+        logger.exception(
+            "Error requesting supplementary data",
+            supplementary_data_url=constructed_supplementary_data_url,
+        )
+        raise SupplementaryDataRequestFailed from exc
+
+    if response.status_code == 200:
+        supplementary_data_response_content = response.content.decode()
+        supplementary_data = json.loads(supplementary_data_response_content)
+
+        return validate_supplementary_data(
+            supplementary_data=supplementary_data,
+            dataset_id=dataset_id,
+            unit_id=unit_id,
+            survey_id=survey_id,
+        )
+
+    logger.error(
+        "got a non-200 response for supplementary data request",
+        status_code=response.status_code,
+        schema_url=constructed_supplementary_data_url,
+    )
+
+    raise SupplementaryDataRequestFailed
+
+
+def validate_supplementary_data(
+    supplementary_data: Mapping, dataset_id: str, unit_id: str, survey_id: str
+) -> dict:
+    try:
+        return validate_supplementary_data_v1(
+            supplementary_data=supplementary_data,
+            dataset_id=dataset_id,
+            unit_id=unit_id,
+            survey_id=survey_id,
+        )
+    except ValidationError as e:
+        raise ValidationError("Invalid supplementary data") from e

app/settings.py

@@ -144,6 +144,7 @@ def utcoffset_or_fail(date_value, key):
 
 SURVEY_TYPE = os.getenv("SURVEY_TYPE", "business")
 
+SDS_API_BASE_URL = os.getenv("SDS_API_BASE_URL")
 
 ACCOUNT_SERVICE_BASE_URL = os.getenv(
     "ACCOUNT_SERVICE_BASE_URL", "https://surveys.ons.gov.uk"

app/submitter/converter_v2.py

@@ -3,7 +3,7 @@
 
 from structlog import get_logger
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.data_models import AnswerStore, ListStore, ProgressStore, QuestionnaireStore
 from app.data_models.metadata_proxy import MetadataProxy, NoMetadataException
 from app.questionnaire.questionnaire_schema import (

app/utilities/metadata_parser_v2.py

@@ -14,7 +14,7 @@
 )
 from structlog import get_logger
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.questionnaire.rules.utils import parse_iso_8601_datetime
 from app.utilities.metadata_validators import DateString, RegionCode, UUIDString
 

app/utilities/request_session.py

@@ -0,0 +1,21 @@
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
+
+
+def get_retryable_session(
+    max_retries, retry_status_codes, backoff_factor
+) -> requests.Session:
+    session = requests.Session()
+
+    retries = Retry(
+        total=max_retries,
+        status_forcelist=retry_status_codes,
+    )  # Codes to retry according to Google Docs https://cloud.google.com/storage/docs/retry-strategy#client-libraries
+
+    retries.backoff_factor = backoff_factor
+
+    session.mount("http://", HTTPAdapter(max_retries=retries))
+    session.mount("https://", HTTPAdapter(max_retries=retries))
+
+    return session

app/utilities/schema.py

@@ -5,9 +5,7 @@
 from pathlib import Path
 from typing import Any
 
-import requests
 from requests import RequestException
-from requests.adapters import HTTPAdapter, Retry
 from structlog import get_logger
 
 from app.data_models.metadata_proxy import MetadataProxy
@@ -16,6 +14,7 @@
     QuestionnaireSchema,
 )
 from app.utilities.json import json_load, json_loads
+from app.utilities.request_session import get_retryable_session
 
 logger = get_logger()
 
@@ -28,7 +27,7 @@
     "phm_0001": [["en", "cy"]],
 }
 
-SCHEMA_REQUEST_MAX_BACKOFF = 0.2
+SCHEMA_REQUEST_BACKOFF_FACTOR = 0.2
 SCHEMA_REQUEST_MAX_RETRIES = 2  # Totals no. of request should be 3. The initial request + SCHEMA_REQUEST_MAX_RETRIES
 SCHEMA_REQUEST_TIMEOUT = 3
 SCHEMA_REQUEST_RETRY_STATUS_CODES = [
@@ -210,18 +209,11 @@ def load_schema_from_url(
 
     constructed_schema_url = f"{schema_url}?language={language_code}"
 
-    session = requests.Session()
-
-    retries = Retry(
-        total=SCHEMA_REQUEST_MAX_RETRIES,
-        status_forcelist=SCHEMA_REQUEST_RETRY_STATUS_CODES,
-    )  # Codes to retry according to Google Docs https://cloud.google.com/storage/docs/retry-strategy#client-libraries
-
-    # Type ignore: MyPy does not recognise BACKOFF_MAX however it is a property, albeit deprecated
-    retries.BACKOFF_MAX = SCHEMA_REQUEST_MAX_BACKOFF  # type: ignore
-
-    session.mount("http://", HTTPAdapter(max_retries=retries))
-    session.mount("https://", HTTPAdapter(max_retries=retries))
+    session = get_retryable_session(
+        max_retries=SCHEMA_REQUEST_MAX_RETRIES,
+        retry_status_codes=SCHEMA_REQUEST_RETRY_STATUS_CODES,
+        backoff_factor=SCHEMA_REQUEST_BACKOFF_FACTOR,
+    )
 
     try:
         req = session.get(constructed_schema_url, timeout=SCHEMA_REQUEST_TIMEOUT)

app/utilities/supplementary_data_parser.py

@@ -0,0 +1,89 @@
+from typing import Mapping
+
+from marshmallow import (
+    INCLUDE,
+    Schema,
+    ValidationError,
+    fields,
+    validate,
+    validates_schema,
+)
+
+from app.authentication.auth_payload_versions import SupplementaryDataSchemaVersion
+from app.utilities.metadata_parser_v2 import VALIDATORS, StripWhitespaceMixin
+
+
+class ItemsSchema(Schema):
+    identifier = VALIDATORS["string"](validate=validate.Length(min=1))
+
+
+class ItemsData(Schema, StripWhitespaceMixin):
+    pass
+
+
+class SupplementaryData(Schema, StripWhitespaceMixin):
+    identifier = VALIDATORS["string"](validate=validate.Length(min=1))
+    schema_version = VALIDATORS["string"](
+        validate=validate.OneOf([SupplementaryDataSchemaVersion.V1.value])
+    )
+    items = fields.Nested(ItemsData, required=False, unknown=INCLUDE)
+
+    @validates_schema()
+    def validate_unit_id(self, data, **kwargs):
+        # pylint: disable=no-self-use, unused-argument
+        if data and data["identifier"] != self.context["unit_id"]:
+            raise ValidationError(
+                "Supplementary data did not return the specified Unit ID"
+            )
+
+
+class SupplementaryDataMetadataSchema(Schema, StripWhitespaceMixin):
+    dataset_id = VALIDATORS["uuid"]()
+    survey_id = VALIDATORS["string"](validate=validate.Length(min=1))
+    data = fields.Nested(
+        SupplementaryData,
+        required=True,
+        unknown=INCLUDE,
+        validate=validate.Length(min=1),
+    )
+
+    @validates_schema()
+    def validate_dataset_and_survey_id(self, data, **kwargs):
+        # pylint: disable=no-self-use, unused-argument
+        if data:
+            if data["dataset_id"] != self.context["dataset_id"]:
+                raise ValidationError(
+                    "Supplementary data did not return the specified Dataset ID"
+                )
+
+            if data["survey_id"] != self.context["survey_id"]:
+                raise ValidationError(
+                    "Supplementary data did not return the specified Survey ID"
+                )
+
+
+def validate_supplementary_data_v1(
+    supplementary_data: Mapping,
+    dataset_id: str,
+    unit_id: str,
+    survey_id: str,
+) -> dict:
+    """Validate claims required for supplementary data"""
+    supplementary_data_metadata_schema = SupplementaryDataMetadataSchema(
+        unknown=INCLUDE
+    )
+    supplementary_data_metadata_schema.context = {
+        "dataset_id": dataset_id,
+        "unit_id": unit_id,
+        "survey_id": survey_id,
+    }
+    validated_supplementary_data = supplementary_data_metadata_schema.load(
+        supplementary_data
+    )
+
+    if supplementary_data_items := supplementary_data.get("data", {}).get("items"):
+        for key, values in supplementary_data_items.items():
+            items = [ItemsSchema(unknown=INCLUDE).load(value) for value in values]
+            validated_supplementary_data["data"]["items"][key] = items
+
+    return validated_supplementary_data

app/views/handlers/feedback.py

@@ -7,7 +7,7 @@
 from sdc.crypto.encrypter import encrypt
 from werkzeug.datastructures import MultiDict
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.data_models import QuestionnaireStore
 from app.data_models.metadata_proxy import MetadataProxy, NoMetadataException
 from app.data_models.session_data import SessionData

app/views/handlers/submission.py

@@ -5,7 +5,7 @@
 from flask import session as cookie_session
 from sdc.crypto.encrypter import encrypt
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.data_models.metadata_proxy import MetadataProxy
 from app.globals import get_session_store
 from app.keys import KEY_PURPOSE_SUBMISSION