Adding request overrides for consent requests

CHANGELOG.md

@@ -21,6 +21,7 @@ The types of changes are:
 - Deprecate LastServedNotice (lastservednoticev2) table [#4910](https://github.com/ethyca/fides/pull/4910)
 - Added erasure support to the Recurly integration [#4891](https://github.com/ethyca/fides/pull/4891)
 - Added UI for configuring integrations for detection/discovery [#4922](https://github.com/ethyca/fides/pull/4922)
+- Request overrides for opt-in and opt-out consent requests [#4920](https://github.com/ethyca/fides/pull/4920)
 
 ### Changed
 - Set default ports for local development of client projects (:3001 for privacy center and :3000 for admin-ui) [#4912](https://github.com/ethyca/fides/pull/4912)

dev-requirements.txt

@@ -10,6 +10,7 @@ pylint==2.15.4
 pytest-asyncio==0.19.0
 pytest-cov==4.0.0
 pytest-env==0.6.2
+pytest-mock==3.14.0
 pytest==7.2.2
 requests-mock==1.10.0
 setuptools>=64.0.2

src/fides/api/service/connectors/saas_connector.py

@@ -604,24 +604,35 @@ def run_consent_request(
         fired: bool = False
         for consent_request in matching_consent_requests:
             self.set_saas_request_state(consent_request)
-            try:
-                prepared_request: SaaSRequestParams = (
-                    query_config.generate_consent_stmt(
-                        policy, privacy_request, consent_request
-                    )
+            # hook for user-provided request override functions
+            if consent_request.request_override:
+                fired = self._invoke_consent_request_override(
+                    consent_request.request_override,
+                    self.create_client(),
+                    policy,
+                    privacy_request,
+                    query_config,
+                    self.secrets,
                 )
-            except ValueError as exc:
-                if consent_request.skip_missing_param_values:
-                    logger.info(
-                        "Skipping optional consent request on node {}: {}",
-                        node.address.value,
-                        exc,
+            else:
+                try:
+                    prepared_request: SaaSRequestParams = (
+                        query_config.generate_consent_stmt(
+                            policy, privacy_request, consent_request
+                        )
                     )
-                    continue
-                raise exc
-            client: AuthenticatedClient = self.create_client()
-            client.send(prepared_request)
-            fired = True
+                except ValueError as exc:
+                    if consent_request.skip_missing_param_values:
+                        logger.info(
+                            "Skipping optional consent request on node {}: {}",
+                            node.address.value,
+                            exc,
+                        )
+                        continue
+                    raise exc
+                client: AuthenticatedClient = self.create_client()
+                client.send(prepared_request)
+                fired = True
         self.unset_connector_state()
         if not fired:
             raise SkippingConsentPropagation(
@@ -683,7 +694,7 @@ def _invoke_test_request_override(
 
         Contains error handling for uncaught exceptions coming out of the override.
         """
-        override_function: Callable[..., Union[List[Row], int, None]] = (
+        override_function: Callable[..., Union[List[Row], int, bool, None]] = (
             SaaSRequestOverrideFactory.get_override(
                 override_function_name, SaaSRequestType.TEST
             )
@@ -716,7 +727,7 @@ def _invoke_read_request_override(
 
         Contains error handling for uncaught exceptions coming out of the override.
         """
-        override_function: Callable[..., Union[List[Row], int, None]] = (
+        override_function: Callable[..., Union[List[Row], int, bool, None]] = (
             SaaSRequestOverrideFactory.get_override(
                 override_function_name, SaaSRequestType.READ
             )
@@ -756,7 +767,7 @@ def _invoke_masking_request_override(
         Includes the necessary data preparations for override input
         and has error handling for uncaught exceptions coming out of the override
         """
-        override_function: Callable[..., Union[List[Row], int, None]] = (
+        override_function: Callable[..., Union[List[Row], int, bool, None]] = (
             SaaSRequestOverrideFactory.get_override(
                 override_function_name, SaaSRequestType(query_config.action)
             )
@@ -786,6 +797,39 @@ def _invoke_masking_request_override(
             )
             raise FidesopsException(str(exc))
 
+    @staticmethod
+    def _invoke_consent_request_override(
+        override_function_name: str,
+        client: AuthenticatedClient,
+        policy: Policy,
+        privacy_request: PrivacyRequest,
+        query_config: SaaSQueryConfig,
+        secrets: Any,
+    ) -> bool:
+        """
+        Invokes the appropriate user-defined SaaS request override for consent requests
+        and performs error handling for uncaught exceptions coming out of the override.
+        """
+        override_function: Callable[..., Union[List[Row], int, bool, None]] = (
+            SaaSRequestOverrideFactory.get_override(
+                override_function_name, SaaSRequestType(query_config.action)
+            )
+        )
+        try:
+            return override_function(
+                client,
+                policy,
+                privacy_request,
+                secrets,
+            )  # type: ignore
+        except Exception as exc:
+            logger.error(
+                "Encountered error executing override consent function '{}",
+                override_function_name,
+                exc_info=True,
+            )
+            raise FidesopsException(str(exc))
+
     def _get_consent_requests_by_preference(self, opt_in: bool) -> List[SaaSRequest]:
         """Helper to either pull out the opt-in requests or the opt out requests that were defined."""
         consent_requests: Optional[ConsentRequestMap] = (

src/fides/api/service/saas_request/saas_request_override_factory.py

@@ -22,6 +22,8 @@ class SaaSRequestType(Enum):
     UPDATE = "update"
     DATA_PROTECTION_REQUEST = "data_protection_request"
     DELETE = "delete"
+    OPT_IN = "opt_in"
+    OPT_OUT = "opt_out"
 
 
 class SaaSRequestOverrideFactory:
@@ -31,7 +33,7 @@ class SaaSRequestOverrideFactory:
     """
 
     registry: Dict[
-        SaaSRequestType, Dict[str, Callable[..., Union[List[Row], int, None]]]
+        SaaSRequestType, Dict[str, Callable[..., Union[List[Row], int, bool, None]]]
     ] = {}
     valid_overrides: Dict[SaaSRequestType, str] = {}
 
@@ -42,8 +44,8 @@ class SaaSRequestOverrideFactory:
 
     @classmethod
     def register(cls, name: str, request_types: List[SaaSRequestType]) -> Callable[
-        [Callable[..., Union[List[Row], int, None]]],
-        Callable[..., Union[List[Row], int, None]],
+        [Callable[..., Union[List[Row], int, bool, None]]],
+        Callable[..., Union[List[Row], int, bool, None]],
     ]:
         """
         Decorator to register the custom-implemented SaaS request override
@@ -58,8 +60,8 @@ def register(cls, name: str, request_types: List[SaaSRequestType]) -> Callable[
             )
 
         def wrapper(
-            override_function: Callable[..., Union[List[Row], int, None]],
-        ) -> Callable[..., Union[List[Row], int, None]]:
+            override_function: Callable[..., Union[List[Row], int, bool, None]],
+        ) -> Callable[..., Union[List[Row], int, bool, None]]:
             for request_type in request_types:
                 logger.debug(
                     "Registering new SaaS request override function '{}' under name '{}' for SaaSRequestType {}",
@@ -79,6 +81,8 @@ def wrapper(
                     SaaSRequestType.DATA_PROTECTION_REQUEST,
                 ):
                     validate_update_override_function(override_function)
+                elif request_type in (SaaSRequestType.OPT_IN, SaaSRequestType.OPT_OUT):
+                    validate_consent_override_function(override_function)
                 else:
                     raise ValueError(
                         f"Invalid SaaSRequestType '{request_type}' provided for SaaS request override function"
@@ -105,14 +109,14 @@ def wrapper(
     @classmethod
     def get_override(
         cls, override_function_name: str, request_type: SaaSRequestType
-    ) -> Callable[..., Union[List[Row], int, None]]:
+    ) -> Callable[..., Union[List[Row], int, bool, None]]:
         """
         Returns the request override function given the name.
         Raises NoSuchSaaSRequestOverrideException if the named override
         does not exist.
         """
         try:
-            override_function: Callable[..., Union[List[Row], int, None]] = (
+            override_function: Callable[..., Union[List[Row], int, bool, None]] = (
                 cls.registry[request_type][override_function_name]
             )
         except KeyError:
@@ -186,5 +190,28 @@ def validate_update_override_function(f: Callable) -> None:
         )
 
 
+def validate_consent_override_function(f: Callable) -> None:
+    """
+    Perform some basic checks on the user-provided SaaS request override function
+    that will be used with `consent` actions.
+
+    The validation is not overly strict to allow for some flexibility in
+    the functions that are used for overrides, but we check to ensure that
+    the function meets the framework's basic expectations.
+
+    Specifically, the validation checks that function's return type is `bool`
+    and that it declares at least 4 parameters.
+    """
+    sig: Signature = signature(f)
+    if sig.return_annotation is not bool:
+        raise InvalidSaaSRequestOverrideException(
+            "Provided SaaS request override function must return a bool"
+        )
+    if len(sig.parameters) < 4:
+        raise InvalidSaaSRequestOverrideException(
+            "Provided SaaS request override function must declare at least 4 parameters"
+        )
+
+
 # TODO: Avoid running this on import?
 register = SaaSRequestOverrideFactory.register

tests/fixtures/saas/recurly_fixtures.py

@@ -15,31 +15,35 @@
 
 secrets = get_secrets("recurly")
 
+
 @pytest.fixture(scope="session")
 def recurly_secrets(saas_config) -> Dict[str, Any]:
     return {
         "domain": pydash.get(saas_config, "recurly.domain") or secrets["domain"],
         "api_key": pydash.get(saas_config, "recurly.api_key") or secrets["api_key"],
     }
 
+
 @pytest.fixture(scope="session")
 def recurly_identity_email(saas_config) -> str:
     return (
         pydash.get(saas_config, "recurly.identity_email") or secrets["identity_email"]
     )
 
+
 @pytest.fixture
 def recurly_erasure_identity_email() -> str:
     return generate_random_email()
 
+
 @pytest.fixture
 def recurly_erasure_data(
     recurly_erasure_identity_email: str,
     recurly_secrets,
 ) -> Generator:
     # setup for adding erasure info, a 'code' is required to add a new user
     gen_string = string.ascii_lowercase
-    code = ''.join(random.choice(gen_string) for i in range(10))
+    code = "".join(random.choice(gen_string) for i in range(10))
 
     base_url = f"https://{recurly_secrets['domain']}"
     auth = HTTPBasicAuth(recurly_secrets["api_key"], None)
@@ -98,9 +102,9 @@ def recurly_erasure_data(
             "city": "Pittsburgh",
             "region": "string",
             "postal_code": "3446",
-            "country": "IN"
+            "country": "IN",
         },
-        "number": "4111 1111 1111 1111"
+        "number": "4111 1111 1111 1111",
     }
     billing_url = f"{accounts_url}/{account_id}/billing_info"
     response = requests.put(
@@ -111,6 +115,7 @@ def recurly_erasure_data(
     )
     assert response.ok
 
+
 @pytest.fixture
 def recurly_runner(db, cache, recurly_secrets) -> ConnectorRunner:
     return ConnectorRunner(db, cache, "recurly", recurly_secrets)