Make response expiry date mandatory (#1104)

app/data_models/metadata_proxy.py

@@ -49,11 +49,11 @@ class MetadataProxy:
     case_id: str
     collection_exercise_sid: str
     response_id: str
+    response_expires_at: datetime
     survey_metadata: Optional[SurveyMetadata] = None
     schema_url: Optional[str] = None
     schema_name: Optional[str] = None
     language_code: Optional[str] = None
-    response_expires_at: Optional[datetime] = None
     channel: Optional[str] = None
     region_code: Optional[str] = None
     version: Optional[AuthPayloadVersion] = None

app/data_models/questionnaire_store.py

@@ -93,12 +93,10 @@ def save(self) -> None:
         collection_exercise_sid = (
             self.collection_exercise_sid or self._metadata["collection_exercise_sid"]
         )
-        response_expires_at = self._metadata.get("response_expires_at")
+        response_expires_at = self._metadata["response_expires_at"]
         self._storage.save(
             data=data,
             collection_exercise_sid=collection_exercise_sid,
             submitted_at=self.submitted_at,
-            expires_at=parse_iso_8601_datetime(response_expires_at)
-            if response_expires_at
-            else None,
+            expires_at=parse_iso_8601_datetime(response_expires_at),
         )

app/utilities/metadata_parser.py

@@ -65,7 +65,7 @@ class RunnerMetadataSchema(Schema, StripWhitespaceMixin):
     )  # type:ignore
     case_type = VALIDATORS["string"](required=False)  # type:ignore
     response_expires_at = VALIDATORS["iso_8601_date_string"](
-        required=False,
+        required=True,
         validate=lambda x: parse_iso_8601_datetime(x) > datetime.now(tz=timezone.utc),
     )  # type:ignore
 

app/utilities/metadata_parser_v2.py

@@ -89,7 +89,7 @@ class RunnerMetadataSchema(Schema, StripWhitespaceMixin):
         required=False, validate=validate.Length(min=1)
     )  # type:ignore
     response_expires_at = VALIDATORS["iso_8601_date_string"](
-        required=False,
+        required=True,
         validate=lambda x: parse_iso_8601_datetime(x) > datetime.now(tz=timezone.utc),
     )  # type:ignore
     region_code = VALIDATORS["string"](

tests/app/data_model/conftest.py

@@ -6,6 +6,7 @@
 from app.data_models.progress_store import CompletionStatus
 from app.data_models.session_store import SessionStore
 from app.storage import storage_encryption
+from tests.app.parser.conftest import get_response_expires_at
 
 
 @pytest.fixture
@@ -81,7 +82,10 @@ def store_to_serialize(answer_store):
 @pytest.fixture
 def basic_input():
     return {
-        "METADATA": {"test": True},
+        "METADATA": {
+            "test": True,
+            "response_expires_at": get_response_expires_at(),
+        },
         "ANSWERS": [{"answer_id": "test", "value": "test"}],
         "LISTS": [],
         "PROGRESS": [

tests/app/data_model/test_metadata_proxy.py

@@ -12,6 +12,7 @@
     "tx_id": "tx_id",
     "collection_exercise_sid": "collection_exercise_sid",
     "case_id": "case_id",
+    "response_expires_at": "2023-04-24T10:46:32+00:00",
 }
 
 METADATA_V2 = {
@@ -22,6 +23,7 @@
     "tx_id": "tx_id",
     "collection_exercise_sid": "collection_exercise_sid",
     "case_id": "case_id",
+    "response_expires_at": "2023-04-24T10:46:32+00:00",
     "survey_metadata": {
         "data": {
             "ru_ref": "432423423423",
@@ -46,6 +48,10 @@
             MetadataProxy.from_dict(METADATA_V2)["schema_name"],
             METADATA_V2["schema_name"],
         ),
+        (
+            MetadataProxy.from_dict(METADATA_V2)["response_expires_at"],
+            METADATA_V2["response_expires_at"],
+        ),
         (MetadataProxy.from_dict(METADATA_V1)["non_existing"], None),
         (MetadataProxy.from_dict(METADATA_V2)["non_existing"], None),
     ),

tests/app/parser/conftest.py

@@ -1,5 +1,6 @@
 # pylint: disable=redefined-outer-name
 import uuid
+from datetime import datetime, timedelta, timezone
 
 import pytest
 
@@ -37,6 +38,7 @@ def fake_metadata_runner():
         "response_id": str(uuid.uuid4()),
         "account_service_url": "https://ras.ons.gov.uk",
         "case_id": str(uuid.uuid4()),
+        "response_expires_at": get_response_expires_at(),
     }
 
 
@@ -48,6 +50,7 @@ def fake_business_metadata_runner():
 
     metadata["eq_id"] = "mbs"
     metadata["form_type"] = "0253"
+    metadata["response_expires_at"] = get_response_expires_at()
 
     return metadata
 
@@ -67,6 +70,7 @@ def fake_metadata_full():
         "return_by": "2016-07-07",
         "case_ref": "1000000000000001",
         "case_id": str(uuid.uuid4()),
+        "response_expires_at": get_response_expires_at(),
     }
 
     return dict(fake_metadata_runner(), **fake_questionnaire_claims)
@@ -84,6 +88,7 @@ def fake_metadata_runner_v2():
         "case_id": str(uuid.uuid4()),
         "version": AuthPayloadVersion.V2.value,
         "survey_metadata": {"data": {"key": "value"}},
+        "response_expires_at": get_response_expires_at(),
     }
 
 
@@ -103,6 +108,7 @@ def fake_metadata_full_v2_business():
         "case_ref": "1000000000000001",
         "ru_ref": "123456789",
         "form_type": "I",
+        "response_expires_at": get_response_expires_at(),
     }
 
     metadata = fake_metadata_runner_v2()
@@ -140,3 +146,7 @@ def fake_questionnaire_metadata_requirements_full():
         {"name": "ref_p_end_date", "type": "string"},
         {"name": "account_service_url", "type": "url", "optional": True},
     ]
+
+
+def get_response_expires_at() -> str:
+    return (datetime.now(tz=timezone.utc) + timedelta(days=1)).isoformat()

tests/app/submitter/conftest.py

@@ -15,6 +15,7 @@
 from app.questionnaire.questionnaire_schema import QuestionnaireSchema
 from app.settings import ACCOUNT_SERVICE_BASE_URL_SOCIAL
 from app.submitter import RabbitMQSubmitter
+from tests.app.parser.conftest import get_response_expires_at
 
 METADATA_V1 = MetadataProxy.from_dict(
     {
@@ -39,6 +40,7 @@
         "display_address": "68 Abingdon Road, Goathill",
         "case_ref": "1000000000000001",
         "jti": str(uuid.uuid4()),
+        "response_expires_at": get_response_expires_at(),
     }
 )
 
@@ -69,6 +71,7 @@
         "region_code": "GB-ENG",
         "channel": "RH",
         "jti": str(uuid.uuid4()),
+        "response_expires_at": get_response_expires_at(),
     }
 )
 

tests/app/views/handlers/conftest.py

@@ -11,6 +11,7 @@
 from app.data_models.session_data import SessionData
 from app.data_models.session_store import SessionStore
 from app.questionnaire import QuestionnaireSchema
+from tests.app.parser.conftest import get_response_expires_at
 
 time_to_freeze = datetime.now(timezone.utc).replace(second=0, microsecond=0)
 tx_id = str(uuid.uuid4())
@@ -39,6 +40,7 @@
 channel = "H"
 case_ref = "1000000000000001"
 region_code = "GB_WLS"
+response_expires_at = get_response_expires_at()
 
 
 @pytest.fixture
@@ -126,6 +128,7 @@ def metadata():
             "region_code": region_code,
             "case_id": case_id,
             "language_code": language_code,
+            "response_expires_at": response_expires_at,
         }
     )
 
@@ -143,6 +146,7 @@ def metadata_v2():
             "channel": channel,
             "region_code": region_code,
             "account_service_url": "account_service_url",
+            "response_expires_at": get_response_expires_at(),
             "survey_metadata": {
                 "data": {
                     "period_id": period_id,
@@ -210,6 +214,7 @@ def mock_questionnaire_store(mocker):
             "schema_name": schema_name,
             "account_service_url": "account_service_url",
             "response_id": "response_id",
+            "response_expires_at": get_response_expires_at(),
         }
     )
     return questionnaire_store
@@ -231,6 +236,7 @@ def mock_questionnaire_store_v2(mocker):
             "channel": channel,
             "region_code": region_code,
             "account_service_url": "account_service_url",
+            "response_expires_at": get_response_expires_at(),
             "survey_metadata": {
                 "data": {
                     "period_id": period_id,

tests/app/views/handlers/test_individual_response_fulfilment_request.py

@@ -14,6 +14,7 @@
     GB_WLS_REGION_CODE,
     IndividualResponseFulfilmentRequest,
 )
+from tests.app.parser.conftest import get_response_expires_at
 
 DUMMY_MOBILE_NUMBER = "07700900258"
 
@@ -27,6 +28,7 @@ def test_sms_fulfilment_request_payload():
         response_id="response_id",
         account_service_url="account_service_url",
         collection_exercise_sid="collection_exercise_sid",
+        response_expires_at=get_response_expires_at(),
     )
 
     fulfilment_request = IndividualResponseFulfilmentRequest(

tests/app/views/handlers/test_question_with_dynamic_answers.py

@@ -8,6 +8,7 @@
 from app.utilities.schema import load_schema_from_name
 from app.views.handlers.question import Question
 
+from ...parser.conftest import get_response_expires_at
 from .conftest import set_storage_data
 
 
@@ -35,6 +36,7 @@ def test_question_with_dynamic_answers(storage, language, mocker):
     questionnaire_store.list_store = ListStore(
         [{"items": ["tUJzGV", "vhECeh"], "name": "supermarkets"}]
     )
+    questionnaire_store.set_metadata({"response_expires_at": get_response_expires_at()})
     schema = load_schema_from_name("test_dynamic_answers_list_source")
 
     mocker.patch(

tests/functional/jwt_helper.js

@@ -90,6 +90,9 @@ export function generateToken(
   const iat = KJUR.jws.IntDate.get("now");
   const exp = KJUR.jws.IntDate.get("now") + 1800;
   const caseId = uuidv4();
+  const currentDate = new Date();
+  currentDate.setUTCDate(currentDate.getUTCDate() + 1);
+  const isoDate = currentDate.toISOString();
 
   if (version === "v2") {
     payload = {
@@ -106,6 +109,7 @@ export function generateToken(
       account_service_url: "http://localhost:8000",
       survey_metadata: getSurveyMetadata(theme, userId, displayAddress, periodId, periodStr),
       version: "v2",
+      response_expires_at: isoDate,
     };
   } else {
     payload = {
@@ -131,6 +135,7 @@ export function generateToken(
       region_code: regionCode,
       language_code: languageCode,
       account_service_url: "http://localhost:8000",
+      response_expires_at: isoDate,
     };
   }
 

tests/integration/create_token.py

@@ -5,6 +5,7 @@
 
 from app.authentication.auth_payload_version import AuthPayloadVersion
 from app.keys import KEY_PURPOSE_AUTHENTICATION
+from tests.app.parser.conftest import get_response_expires_at
 
 ACCOUNT_SERVICE_URL = "http://upstream.url"
 
@@ -88,6 +89,7 @@ def _get_payload_with_params(
         payload_vars["exp"] = payload_vars["iat"] + float(3600)  # one hour from now
         payload_vars["jti"] = str(uuid4())
         payload_vars["case_id"] = str(uuid4())
+        payload_vars["response_expires_at"] = get_response_expires_at()
 
         for key, value in extra_payload.items():
             payload_vars[key] = value

tests/integration/routes/test_errors.py

@@ -6,6 +6,7 @@
     ACCOUNT_SERVICE_BASE_URL_SOCIAL,
     ONS_URL,
 )
+from tests.app.parser.conftest import get_response_expires_at
 from tests.integration.create_token import ACCOUNT_SERVICE_URL
 from tests.integration.integration_test_case import IntegrationTestCase
 
@@ -31,6 +32,7 @@ class TestErrors(IntegrationTestCase):  # pylint: disable=too-many-public-method
         "language_code": "en",
         "account_service_url": "http://correct.place",
         "roles": [],
+        "response_expires_at": get_response_expires_at(),
     }
 
     def test_errors_404(self):

tests/integration/routes/test_jwt_authentication.py

@@ -13,6 +13,7 @@
     TEST_DO_NOT_USE_SR_PUBLIC_KEY,
     TEST_DO_NOT_USE_UPSTREAM_PRIVATE_KEY,
 )
+from tests.app.parser.conftest import get_response_expires_at
 from tests.integration.app_context_test_case import AppContextTestCase
 from tests.integration.integration_test_case import (
     EQ_USER_AUTHENTICATION_RRM_PRIVATE_KEY_KID,
@@ -85,6 +86,7 @@ def create_payload():
             "ru_name": "Test",
             "return_by": "2016-09-09",
             "account_service_url": "http://upstream.url/",
+            "response_expires_at": get_response_expires_at(),
         }
 
 

tests/integration/test_flush_data.py

@@ -3,6 +3,7 @@
 
 from mock import patch
 
+from tests.app.parser.conftest import get_response_expires_at
 from tests.integration.integration_test_case import IntegrationTestCase
 
 
@@ -179,6 +180,7 @@ def test_flush_data_successful_v2(
                 "lists": [],
             },
             "started_at": "2023-02-07T11:42:32.380784+00:00",
+            "response_expires_at": get_response_expires_at(),
         }
         self.launchSurveyV2("test_textfield")
         form_data = {"name-answer": "Joe Bloggs"}