Integrate with SDS endpoint to support loading Prepop data

.schemas-version

@@ -1 +1 @@
-v3.55.0
+v3.57.0

app/authentication/auth_payload_version.py → app/authentication/auth_payload_versions.py

@@ -3,3 +3,7 @@
 
 class AuthPayloadVersion(Enum):
     V2 = "v2"
+
+
+class SupplementaryDataSchemaVersion(Enum):
+    V1 = "v1"

app/data_models/metadata_proxy.py

@@ -7,7 +7,7 @@
 
 from werkzeug.datastructures import ImmutableDict
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.utilities.make_immutable import make_immutable
 
 
@@ -49,11 +49,11 @@ class MetadataProxy:
     case_id: str
     collection_exercise_sid: str
     response_id: str
+    response_expires_at: datetime
     survey_metadata: Optional[SurveyMetadata] = None
     schema_url: Optional[str] = None
     schema_name: Optional[str] = None
     language_code: Optional[str] = None
-    response_expires_at: Optional[datetime] = None
     channel: Optional[str] = None
     region_code: Optional[str] = None
     version: Optional[AuthPayloadVersion] = None

app/data_models/questionnaire_store.py

@@ -93,12 +93,10 @@ def save(self) -> None:
         collection_exercise_sid = (
             self.collection_exercise_sid or self._metadata["collection_exercise_sid"]
         )
-        response_expires_at = self._metadata.get("response_expires_at")
+        response_expires_at = self._metadata["response_expires_at"]
         self._storage.save(
             data=data,
             collection_exercise_sid=collection_exercise_sid,
             submitted_at=self.submitted_at,
-            expires_at=parse_iso_8601_datetime(response_expires_at)
-            if response_expires_at
-            else None,
+            expires_at=parse_iso_8601_datetime(response_expires_at),
         )

app/routes/flush.py

@@ -7,7 +7,7 @@
 from sdc.crypto.key_store import KeyStore
 from structlog import contextvars, get_logger
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.authentication.user import User
 from app.authentication.user_id_generator import UserIDGenerator
 from app.data_models import QuestionnaireStore
@@ -55,7 +55,14 @@ def flush_data() -> Response:
         user = _get_user(decrypted_token["response_id"])
 
         if metadata := get_metadata(user):
-            contextvars.bind_contextvars(tx_id=metadata.tx_id)
+            contextvars.bind_contextvars(
+                tx_id=metadata.tx_id,
+                ce_id=metadata.collection_exercise_sid,
+            )
+            if schema_name := metadata.schema_name:
+                contextvars.bind_contextvars(schema_name=schema_name)
+            if schema_url := metadata.schema_url:
+                contextvars.bind_contextvars(schema_url=schema_url)
         if _submit_data(user):
             return Response(status=200)
         return Response(status=404)

app/routes/session.py

@@ -1,17 +1,21 @@
+import json
 from datetime import datetime, timezone
 from typing import Any, Iterable, Mapping
 
+import requests
 from flask import Blueprint, g, jsonify, redirect, request
 from flask import session as cookie_session
 from flask import url_for
 from flask_login import login_required, logout_user
 from marshmallow import INCLUDE, ValidationError
+from requests import RequestException
+from requests.adapters import HTTPAdapter, Retry
 from sdc.crypto.exceptions import InvalidTokenException
 from structlog import contextvars, get_logger
 from werkzeug.exceptions import Unauthorized
 from werkzeug.wrappers.response import Response
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.authentication.authenticator import decrypt_token, store_session
 from app.authentication.jti_claim_storage import JtiTokenUsed, use_jti_claim
 from app.data_models.metadata_proxy import MetadataProxy
@@ -29,11 +33,30 @@
     validate_runner_claims_v2,
 )
 from app.utilities.schema import load_schema_from_metadata
+from app.utilities.supplementary_data_parser import validate_supplementary_data_v1
 
 logger = get_logger()
 
 session_blueprint = Blueprint("session", __name__)
 
+SUPPLEMENTARY_DATA_URL = "http://localhost:5003/v1/unit_data"
+SUPPLEMENTARY_DATA_REQUEST_MAX_BACKOFF = 0.2
+SUPPLEMENTARY_DATA_REQUEST_MAX_RETRIES = 2  # Totals no. of request should be 3. The initial request + SUPPLEMENTARY_DATA_REQUEST_MAX_RETRIES
+SUPPLEMENTARY_DATA_REQUEST_TIMEOUT = 3
+SUPPLEMENTARY_DATA_REQUEST_RETRY_STATUS_CODES = [
+    408,
+    429,
+    500,
+    502,
+    503,
+    504,
+]
+
+
+class SupplementaryDataRequestFailed(Exception):
+    def __str__(self) -> str:
+        return "Supplementary Data request failed"
+
 
 @session_blueprint.after_request
 def add_cache_control(response: Response) -> Response:
@@ -128,9 +151,76 @@ def login() -> Response:
 
     cookie_session["language_code"] = metadata.language_code
 
+    if (dataset_id := metadata["sds_dataset_id"]) and ru_ref:
+        get_supplementary_data(
+            supplementary_data_url=SUPPLEMENTARY_DATA_URL,
+            dataset_id=dataset_id,
+            ru_ref=ru_ref,
+        )
+
     return redirect(url_for("questionnaire.get_questionnaire"))
 
 
+def get_supplementary_data(
+    supplementary_data_url: str, dataset_id: str, ru_ref: str
+) -> dict:
+    constructed_supplementary_data_url = (
+        f"{supplementary_data_url}?dataset_id={dataset_id}&unit_id={ru_ref}"
+    )
+
+    session = requests.Session()
+
+    retries = Retry(
+        total=SUPPLEMENTARY_DATA_REQUEST_MAX_RETRIES,
+        status_forcelist=SUPPLEMENTARY_DATA_REQUEST_RETRY_STATUS_CODES,
+    )  # Codes to retry according to Google Docs https://cloud.google.com/storage/docs/retry-strategy#client-libraries
+
+    # Type ignore: MyPy does not recognise BACKOFF_MAX however it is a property, albeit deprecated
+    retries.BACKOFF_MAX = SUPPLEMENTARY_DATA_REQUEST_MAX_BACKOFF  # type: ignore
+
+    session.mount("http://", HTTPAdapter(max_retries=retries))
+    session.mount("https://", HTTPAdapter(max_retries=retries))
+
+    try:
+        response = session.get(
+            constructed_supplementary_data_url,
+            timeout=SUPPLEMENTARY_DATA_REQUEST_TIMEOUT,
+        )
+    except RequestException as exc:
+        logger.exception(
+            "Error requesting supplementary data",
+            supplementary_data_url=constructed_supplementary_data_url,
+        )
+        raise SupplementaryDataRequestFailed from exc
+
+    if response.status_code == 200:
+        supplementary_data_response_content = response.content.decode()
+        supplementary_data = json.loads(supplementary_data_response_content)
+
+        return validate_supplementary_data(
+            supplementary_data=supplementary_data, dataset_id=dataset_id, ru_ref=ru_ref
+        )
+
+    logger.error(
+        "got a non-200 response for supplementary data request",
+        status_code=response.status_code,
+        schema_url=constructed_supplementary_data_url,
+    )
+
+    raise SupplementaryDataRequestFailed
+
+
+def validate_supplementary_data(
+    supplementary_data: Mapping, dataset_id: str, ru_ref: str
+) -> dict:
+    try:
+        return validate_supplementary_data_v1(
+            supplementary_data=supplementary_data, dataset_id=dataset_id, ru_ref=ru_ref
+        )
+    except ValidationError as e:
+        raise ValidationError("Invalid supplementary_dataulation data") from e
+
+
 def validate_jti(decrypted_token: dict[str, str | list | int]) -> None:
     # Type ignore: decrypted_token["exp"] will return a valid timestamp with compatible typing
     expires_at = datetime.fromtimestamp(decrypted_token["exp"], tz=timezone.utc)  # type: ignore

app/submitter/converter_v2.py

@@ -3,7 +3,7 @@
 
 from structlog import get_logger
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.data_models import AnswerStore, ListStore, ProgressStore, QuestionnaireStore
 from app.data_models.metadata_proxy import MetadataProxy, NoMetadataException
 from app.questionnaire.questionnaire_schema import (

app/utilities/metadata_parser.py

@@ -65,7 +65,7 @@ class RunnerMetadataSchema(Schema, StripWhitespaceMixin):
     )  # type:ignore
     case_type = VALIDATORS["string"](required=False)  # type:ignore
     response_expires_at = VALIDATORS["iso_8601_date_string"](
-        required=False,
+        required=True,
         validate=lambda x: parse_iso_8601_datetime(x) > datetime.now(tz=timezone.utc),
     )  # type:ignore
 

app/utilities/metadata_parser_v2.py

@@ -14,7 +14,7 @@
 )
 from structlog import get_logger
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.questionnaire.rules.utils import parse_iso_8601_datetime
 from app.utilities.metadata_validators import DateString, RegionCode, UUIDString
 
@@ -89,7 +89,7 @@ class RunnerMetadataSchema(Schema, StripWhitespaceMixin):
         required=False, validate=validate.Length(min=1)
     )  # type:ignore
     response_expires_at = VALIDATORS["iso_8601_date_string"](
-        required=False,
+        required=True,
         validate=lambda x: parse_iso_8601_datetime(x) > datetime.now(tz=timezone.utc),
     )  # type:ignore
     region_code = VALIDATORS["string"](

app/utilities/supplementary_data_parser.py

@@ -0,0 +1,88 @@
+from typing import Mapping
+
+from marshmallow import (
+    EXCLUDE,
+    INCLUDE,
+    Schema,
+    ValidationError,
+    fields,
+    validate,
+    validates_schema,
+)
+
+from app.authentication.auth_payload_versions import SupplementaryDataSchemaVersion
+from app.utilities.metadata_parser_v2 import VALIDATORS, StripWhitespaceMixin
+
+
+class ItemsSchema(Schema):
+    identifier = VALIDATORS["string"](validate=validate.Length(min=1))
+
+
+class ItemsData(Schema, StripWhitespaceMixin):
+    pass
+
+
+class SupplementaryData(Schema, StripWhitespaceMixin):
+    identifier = VALIDATORS["string"](validate=validate.Length(min=1))
+    schema_version = VALIDATORS["string"](
+        validate=validate.OneOf([SupplementaryDataSchemaVersion.V1.value])
+    )
+    items = fields.Nested(ItemsData, required=False, unknown=EXCLUDE)
+
+    @validates_schema()
+    def validate_unit_id(self, data, **kwargs):
+        # pylint: disable=no-self-use, unused-argument
+        if (
+            data
+            and data.get("identifier")
+            and data["identifier"] != self.context["ru_ref"]
+        ):
+            raise ValidationError(
+                "Supplementary data did not return the specified Unit ID"
+            )
+
+
+class SupplementaryDataMetadataSchema(Schema, StripWhitespaceMixin):
+    dataset_id = VALIDATORS["string"](validate=validate.Length(min=1))
+    survey_id = VALIDATORS["string"](validate=validate.Length(min=1))
+    data = fields.Nested(
+        SupplementaryData,
+        required=True,
+        unknown=EXCLUDE,
+        validate=validate.Length(min=1),
+    )
+
+    @validates_schema()
+    def validate_dataset_id(self, data, **kwargs):
+        # pylint: disable=no-self-use, unused-argument
+        if (
+            data
+            and data.get("dataset_id")
+            and data["dataset_id"] != self.context["dataset_id"]
+        ):
+            raise ValidationError(
+                "Supplementary data did not return the specified Dataset ID"
+            )
+
+
+def validate_supplementary_data_v1(
+    supplementary_data: Mapping, dataset_id: str, ru_ref: str
+) -> dict:
+    """Validate claims required for runner to function"""
+    supplementary_data_metadata_schema = SupplementaryDataMetadataSchema(
+        unknown=INCLUDE
+    )
+    supplementary_data_metadata_schema.context = {
+        "dataset_id": dataset_id,
+        "ru_ref": ru_ref,
+    }
+    validated_supplementary_data = supplementary_data_metadata_schema.load(
+        supplementary_data
+    )
+
+    if supplementary_data_items := supplementary_data.get("data", {}).get("items"):
+        for key, values in supplementary_data_items.items():
+            items = [ItemsSchema(unknown=INCLUDE).load(value) for value in values]
+            validated_supplementary_data["data"]["items"][key] = items
+
+    return validated_supplementary_data

app/views/handlers/feedback.py

@@ -7,7 +7,7 @@
 from sdc.crypto.encrypter import encrypt
 from werkzeug.datastructures import MultiDict
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.data_models import QuestionnaireStore
 from app.data_models.metadata_proxy import MetadataProxy, NoMetadataException
 from app.data_models.session_data import SessionData

app/views/handlers/submission.py

@@ -5,7 +5,7 @@
 from flask import session as cookie_session
 from sdc.crypto.encrypter import encrypt
 
-from app.authentication.auth_payload_version import AuthPayloadVersion
+from app.authentication.auth_payload_versions import AuthPayloadVersion
 from app.data_models.metadata_proxy import MetadataProxy
 from app.globals import get_session_store
 from app.keys import KEY_PURPOSE_SUBMISSION