diff --git a/CHANGELOG.md b/CHANGELOG.md index 9403a4087d..fe3abc5a6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The types of changes are: - Deprecate LastServedNotice (lastservednoticev2) table [#4910](https://github.com/ethyca/fides/pull/4910) - Added erasure support to the Recurly integration [#4891](https://github.com/ethyca/fides/pull/4891) - Added UI for configuring integrations for detection/discovery [#4922](https://github.com/ethyca/fides/pull/4922) +- Request overrides for opt-in and opt-out consent requests [#4920](https://github.com/ethyca/fides/pull/4920) ### Changed - Set default ports for local development of client projects (:3001 for privacy center and :3000 for admin-ui) [#4912](https://github.com/ethyca/fides/pull/4912) diff --git a/dev-requirements.txt b/dev-requirements.txt index 7cedd99b32..b6e1c69ba0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -10,6 +10,7 @@ pylint==2.15.4 pytest-asyncio==0.19.0 pytest-cov==4.0.0 pytest-env==0.6.2 +pytest-mock==3.14.0 pytest==7.2.2 requests-mock==1.10.0 setuptools>=64.0.2 diff --git a/src/fides/api/service/connectors/saas_connector.py b/src/fides/api/service/connectors/saas_connector.py index 4b47ecdc58..19dcb38738 100644 --- a/src/fides/api/service/connectors/saas_connector.py +++ b/src/fides/api/service/connectors/saas_connector.py @@ -604,24 +604,35 @@ def run_consent_request( fired: bool = False for consent_request in matching_consent_requests: self.set_saas_request_state(consent_request) - try: - prepared_request: SaaSRequestParams = ( - query_config.generate_consent_stmt( - policy, privacy_request, consent_request - ) + # hook for user-provided request override functions + if consent_request.request_override: + fired = self._invoke_consent_request_override( + consent_request.request_override, + self.create_client(), + policy, + privacy_request, + query_config, + self.secrets, ) - except ValueError as exc: - if consent_request.skip_missing_param_values: - logger.info( - "Skipping optional consent request on node {}: {}", - node.address.value, - exc, + else: + try: + prepared_request: SaaSRequestParams = ( + query_config.generate_consent_stmt( + policy, privacy_request, consent_request + ) ) - continue - raise exc - client: AuthenticatedClient = self.create_client() - client.send(prepared_request) - fired = True + except ValueError as exc: + if consent_request.skip_missing_param_values: + logger.info( + "Skipping optional consent request on node {}: {}", + node.address.value, + exc, + ) + continue + raise exc + client: AuthenticatedClient = self.create_client() + client.send(prepared_request) + fired = True self.unset_connector_state() if not fired: raise SkippingConsentPropagation( @@ -683,7 +694,7 @@ def _invoke_test_request_override( Contains error handling for uncaught exceptions coming out of the override. """ - override_function: Callable[..., Union[List[Row], int, None]] = ( + override_function: Callable[..., Union[List[Row], int, bool, None]] = ( SaaSRequestOverrideFactory.get_override( override_function_name, SaaSRequestType.TEST ) @@ -716,7 +727,7 @@ def _invoke_read_request_override( Contains error handling for uncaught exceptions coming out of the override. """ - override_function: Callable[..., Union[List[Row], int, None]] = ( + override_function: Callable[..., Union[List[Row], int, bool, None]] = ( SaaSRequestOverrideFactory.get_override( override_function_name, SaaSRequestType.READ ) @@ -756,7 +767,7 @@ def _invoke_masking_request_override( Includes the necessary data preparations for override input and has error handling for uncaught exceptions coming out of the override """ - override_function: Callable[..., Union[List[Row], int, None]] = ( + override_function: Callable[..., Union[List[Row], int, bool, None]] = ( SaaSRequestOverrideFactory.get_override( override_function_name, SaaSRequestType(query_config.action) ) @@ -786,6 +797,39 @@ def _invoke_masking_request_override( ) raise FidesopsException(str(exc)) + @staticmethod + def _invoke_consent_request_override( + override_function_name: str, + client: AuthenticatedClient, + policy: Policy, + privacy_request: PrivacyRequest, + query_config: SaaSQueryConfig, + secrets: Any, + ) -> bool: + """ + Invokes the appropriate user-defined SaaS request override for consent requests + and performs error handling for uncaught exceptions coming out of the override. + """ + override_function: Callable[..., Union[List[Row], int, bool, None]] = ( + SaaSRequestOverrideFactory.get_override( + override_function_name, SaaSRequestType(query_config.action) + ) + ) + try: + return override_function( + client, + policy, + privacy_request, + secrets, + ) # type: ignore + except Exception as exc: + logger.error( + "Encountered error executing override consent function '{}", + override_function_name, + exc_info=True, + ) + raise FidesopsException(str(exc)) + def _get_consent_requests_by_preference(self, opt_in: bool) -> List[SaaSRequest]: """Helper to either pull out the opt-in requests or the opt out requests that were defined.""" consent_requests: Optional[ConsentRequestMap] = ( diff --git a/src/fides/api/service/saas_request/override_implementations/mailchimp_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/mailchimp_request_overrides.py deleted file mode 100644 index e279a3adbc..0000000000 --- a/src/fides/api/service/saas_request/override_implementations/mailchimp_request_overrides.py +++ /dev/null @@ -1,106 +0,0 @@ -from json import dumps -from typing import Any, Dict, List - -import pydash - -from fides.api.graph.execution import ExecutionNode -from fides.api.models.policy import Policy -from fides.api.models.privacy_request import PrivacyRequest -from fides.api.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient -from fides.api.service.saas_request.saas_request_override_factory import ( - SaaSRequestType, - register, -) -from fides.api.util.collection_util import Row - - -@register("mailchimp_messages_access", [SaaSRequestType.READ]) -def mailchimp_messages_access( - client: AuthenticatedClient, - node: ExecutionNode, - policy: Policy, - privacy_request: PrivacyRequest, - input_data: Dict[str, List[Any]], - secrets: Dict[str, Any], -) -> List[Row]: - """ - Equivalent SaaS config for the code in this function. - - Request params still need to be defined for endpoints with overrides. - This is to provide the necessary reference and identity data as part - of graph traversal. The resulting values are passed in as parameters - so we don't need to define the data retrieval here. - - path: /3.0/conversations//messages - request_params: - - name: conversation_id - type: path - references: - - dataset: mailchimp_instance - field: conversations.id - direction: from - data_path: conversation_messages - postprocessors: - - strategy: filter - configuration: - field: from_email - value: - identity: email - """ - # gather request params - conversation_ids = input_data.get("conversation_id") - - # build and execute request for each input data value - processed_data = [] - if conversation_ids: - for conversation_id in conversation_ids: - response = client.send( - SaaSRequestParams( - method=HTTPMethod.GET, - path=f"/3.0/conversations/{conversation_id}/messages", - ) - ) - - # unwrap and post-process response - response_data = pydash.get(response.json(), "conversation_messages") - filtered_data = pydash.filter_( - response_data, - {"from_email": privacy_request.get_cached_identity_data().get("email")}, - ) - - # build up final result - processed_data.extend(filtered_data) - - return processed_data - - -@register("mailchimp_member_update", [SaaSRequestType.UPDATE]) -def mailchimp_member_update( - client: AuthenticatedClient, - param_values_per_row: List[Dict[str, Any]], - policy: Policy, - privacy_request: PrivacyRequest, - secrets: Dict[str, Any], -) -> int: - rows_updated = 0 - # each update_params dict correspond to a record that needs to be updated - for row_param_values in param_values_per_row: - # get params to be used in update request - list_id = row_param_values.get("list_id") - subscriber_hash = row_param_values.get("subscriber_hash") - - # in this case, we can just put the masked object fields object - # directly into the request body - update_body = dumps(row_param_values["masked_object_fields"]) - - client.send( - SaaSRequestParams( - method=HTTPMethod.PUT, - path=f"/3.0/lists/{list_id}/members/{subscriber_hash}", - body=update_body, - ) - ) - - rows_updated += 1 - return rows_updated diff --git a/src/fides/api/service/saas_request/saas_request_override_factory.py b/src/fides/api/service/saas_request/saas_request_override_factory.py index 44c1292635..01c9dbd152 100644 --- a/src/fides/api/service/saas_request/saas_request_override_factory.py +++ b/src/fides/api/service/saas_request/saas_request_override_factory.py @@ -22,6 +22,8 @@ class SaaSRequestType(Enum): UPDATE = "update" DATA_PROTECTION_REQUEST = "data_protection_request" DELETE = "delete" + OPT_IN = "opt_in" + OPT_OUT = "opt_out" class SaaSRequestOverrideFactory: @@ -31,7 +33,7 @@ class SaaSRequestOverrideFactory: """ registry: Dict[ - SaaSRequestType, Dict[str, Callable[..., Union[List[Row], int, None]]] + SaaSRequestType, Dict[str, Callable[..., Union[List[Row], int, bool, None]]] ] = {} valid_overrides: Dict[SaaSRequestType, str] = {} @@ -42,8 +44,8 @@ class SaaSRequestOverrideFactory: @classmethod def register(cls, name: str, request_types: List[SaaSRequestType]) -> Callable[ - [Callable[..., Union[List[Row], int, None]]], - Callable[..., Union[List[Row], int, None]], + [Callable[..., Union[List[Row], int, bool, None]]], + Callable[..., Union[List[Row], int, bool, None]], ]: """ Decorator to register the custom-implemented SaaS request override @@ -58,8 +60,8 @@ def register(cls, name: str, request_types: List[SaaSRequestType]) -> Callable[ ) def wrapper( - override_function: Callable[..., Union[List[Row], int, None]], - ) -> Callable[..., Union[List[Row], int, None]]: + override_function: Callable[..., Union[List[Row], int, bool, None]], + ) -> Callable[..., Union[List[Row], int, bool, None]]: for request_type in request_types: logger.debug( "Registering new SaaS request override function '{}' under name '{}' for SaaSRequestType {}", @@ -79,6 +81,8 @@ def wrapper( SaaSRequestType.DATA_PROTECTION_REQUEST, ): validate_update_override_function(override_function) + elif request_type in (SaaSRequestType.OPT_IN, SaaSRequestType.OPT_OUT): + validate_consent_override_function(override_function) else: raise ValueError( f"Invalid SaaSRequestType '{request_type}' provided for SaaS request override function" @@ -105,14 +109,14 @@ def wrapper( @classmethod def get_override( cls, override_function_name: str, request_type: SaaSRequestType - ) -> Callable[..., Union[List[Row], int, None]]: + ) -> Callable[..., Union[List[Row], int, bool, None]]: """ Returns the request override function given the name. Raises NoSuchSaaSRequestOverrideException if the named override does not exist. """ try: - override_function: Callable[..., Union[List[Row], int, None]] = ( + override_function: Callable[..., Union[List[Row], int, bool, None]] = ( cls.registry[request_type][override_function_name] ) except KeyError: @@ -186,5 +190,28 @@ def validate_update_override_function(f: Callable) -> None: ) +def validate_consent_override_function(f: Callable) -> None: + """ + Perform some basic checks on the user-provided SaaS request override function + that will be used with `consent` actions. + + The validation is not overly strict to allow for some flexibility in + the functions that are used for overrides, but we check to ensure that + the function meets the framework's basic expectations. + + Specifically, the validation checks that function's return type is `bool` + and that it declares at least 4 parameters. + """ + sig: Signature = signature(f) + if sig.return_annotation is not bool: + raise InvalidSaaSRequestOverrideException( + "Provided SaaS request override function must return a bool" + ) + if len(sig.parameters) < 4: + raise InvalidSaaSRequestOverrideException( + "Provided SaaS request override function must declare at least 4 parameters" + ) + + # TODO: Avoid running this on import? register = SaaSRequestOverrideFactory.register diff --git a/tests/fixtures/saas/recurly_fixtures.py b/tests/fixtures/saas/recurly_fixtures.py index c32f287552..be93438b21 100644 --- a/tests/fixtures/saas/recurly_fixtures.py +++ b/tests/fixtures/saas/recurly_fixtures.py @@ -15,6 +15,7 @@ secrets = get_secrets("recurly") + @pytest.fixture(scope="session") def recurly_secrets(saas_config) -> Dict[str, Any]: return { @@ -22,16 +23,19 @@ def recurly_secrets(saas_config) -> Dict[str, Any]: "api_key": pydash.get(saas_config, "recurly.api_key") or secrets["api_key"], } + @pytest.fixture(scope="session") def recurly_identity_email(saas_config) -> str: return ( pydash.get(saas_config, "recurly.identity_email") or secrets["identity_email"] ) + @pytest.fixture def recurly_erasure_identity_email() -> str: return generate_random_email() + @pytest.fixture def recurly_erasure_data( recurly_erasure_identity_email: str, @@ -39,7 +43,7 @@ def recurly_erasure_data( ) -> Generator: # setup for adding erasure info, a 'code' is required to add a new user gen_string = string.ascii_lowercase - code = ''.join(random.choice(gen_string) for i in range(10)) + code = "".join(random.choice(gen_string) for i in range(10)) base_url = f"https://{recurly_secrets['domain']}" auth = HTTPBasicAuth(recurly_secrets["api_key"], None) @@ -98,9 +102,9 @@ def recurly_erasure_data( "city": "Pittsburgh", "region": "string", "postal_code": "3446", - "country": "IN" + "country": "IN", }, - "number": "4111 1111 1111 1111" + "number": "4111 1111 1111 1111", } billing_url = f"{accounts_url}/{account_id}/billing_info" response = requests.put( @@ -111,6 +115,7 @@ def recurly_erasure_data( ) assert response.ok + @pytest.fixture def recurly_runner(db, cache, recurly_secrets) -> ConnectorRunner: return ConnectorRunner(db, cache, "recurly", recurly_secrets) diff --git a/tests/fixtures/saas/test_data/saas_consent_request_override_config.yml b/tests/fixtures/saas/test_data/saas_consent_request_override_config.yml new file mode 100644 index 0000000000..1abe845327 --- /dev/null +++ b/tests/fixtures/saas/test_data/saas_consent_request_override_config.yml @@ -0,0 +1,31 @@ +saas_config: + fides_key: consent_request_override_example + name: Consent request override example config + type: consent_request_override_example + description: A sample schema used to test consent request overrides + version: 0.0.1 + + connector_params: + - name: domain + - name: api_token + label: API token + + client_config: + protocol: http + host: + authentication: + strategy: bearer + configuration: + token: + + test_request: + method: GET + path: / + + consent_requests: + opt_in: + request_override: opt_in_request_override + opt_out: + request_override: opt_out_request_override + + endpoints: [] diff --git a/tests/fixtures/saas/test_data/saas_consent_request_override_dataset.yml b/tests/fixtures/saas/test_data/saas_consent_request_override_dataset.yml new file mode 100644 index 0000000000..85317bd91d --- /dev/null +++ b/tests/fixtures/saas/test_data/saas_consent_request_override_dataset.yml @@ -0,0 +1,4 @@ +dataset: + - fides_key: consent_request_override_example + name: Consent request override example dataset + collections: [] diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index db965abaa8..7b3859ac01 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -736,3 +736,75 @@ def saas_async_example_connection_config( ) yield connection_config connection_config.delete(db) + + +@pytest.fixture +def saas_consent_request_override_config() -> Dict: + return load_config( + "tests/fixtures/saas/test_data/saas_consent_request_override_config.yml" + ) + + +@pytest.fixture +def saas_consent_request_override_dataset() -> Dict: + return load_dataset( + "tests/fixtures/saas/test_data/saas_consent_request_override_dataset.yml" + )[0] + + +@pytest.fixture(scope="function") +def saas_consent_request_override_secrets(): + return { + "domain": "domain", + "api_key": "api_key", + } + + +@pytest.fixture +def saas_consent_request_override_dataset_config( + db: Session, + saas_consent_request_override_connection_config: ConnectionConfig, + saas_consent_request_override_dataset: Dict, +) -> Generator: + fides_key = saas_consent_request_override_dataset["fides_key"] + saas_consent_request_override_connection_config.name = fides_key + saas_consent_request_override_connection_config.key = fides_key + saas_consent_request_override_connection_config.save(db=db) + + ctl_dataset = CtlDataset.create_from_dataset_dict( + db, saas_consent_request_override_dataset + ) + + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": saas_consent_request_override_connection_config.id, + "fides_key": fides_key, + "ctl_dataset_id": ctl_dataset.id, + }, + ) + yield dataset + dataset.delete(db=db) + ctl_dataset.delete(db) + + +@pytest.fixture(scope="function") +def saas_consent_request_override_connection_config( + db: Session, + saas_consent_request_override_config: Dict[str, Any], + saas_consent_request_override_secrets: Dict[str, Any], +) -> Generator: + fides_key = saas_consent_request_override_config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": saas_consent_request_override_secrets, + "saas_config": saas_consent_request_override_config, + }, + ) + yield connection_config + connection_config.delete(db) diff --git a/tests/ops/integration_tests/saas/connector_runner.py b/tests/ops/integration_tests/saas/connector_runner.py index ff1d81bde8..83d03a2b94 100644 --- a/tests/ops/integration_tests/saas/connector_runner.py +++ b/tests/ops/integration_tests/saas/connector_runner.py @@ -261,7 +261,9 @@ async def new_consent_request( identity = Identity(**identities) privacy_request.cache_identity(identity) - _privacy_preference_history(self.db, privacy_request, identities, opt_in=True) + create_privacy_preference_history( + self.db, privacy_request, identities, opt_in=True + ) opt_in = consent_runner_tester( privacy_request, consent_policy, @@ -271,7 +273,9 @@ async def new_consent_request( self.db, ) - _privacy_preference_history(self.db, privacy_request, identities, opt_in=False) + create_privacy_preference_history( + self.db, privacy_request, identities, opt_in=False + ) opt_out = consent_runner_tester( privacy_request, consent_policy, @@ -462,9 +466,10 @@ def dataset_config( return dataset -def _privacy_preference_history( +def create_privacy_preference_history( db, privacy_request: PrivacyRequest, identities: Dict[str, Any], opt_in: bool ): + """Creates a privacy preference history entry and associates it with the given privacy request and identities""" privacy_notice = PrivacyNotice.create( db=db, data={ @@ -484,7 +489,7 @@ def _privacy_preference_history( ) email_identity = identities["email"] - provided_identity = ProvidedIdentity.create( + ProvidedIdentity.create( db, data={ "privacy_request_id": None, diff --git a/tests/ops/integration_tests/saas/request_override/test_consent_request_override_task.py b/tests/ops/integration_tests/saas/request_override/test_consent_request_override_task.py new file mode 100644 index 0000000000..6764fd1ff5 --- /dev/null +++ b/tests/ops/integration_tests/saas/request_override/test_consent_request_override_task.py @@ -0,0 +1,147 @@ +import random +from typing import Any, Dict + +import pytest + +from fides.api.models.policy import Policy +from fides.api.models.privacy_request import PrivacyRequest, PrivacyRequestStatus +from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient +from fides.api.service.privacy_request.request_runner_service import ( + build_consent_dataset_graph, +) +from fides.api.service.saas_request.saas_request_override_factory import ( + SaaSRequestOverrideFactory, + SaaSRequestType, + register, +) +from tests.conftest import consent_runner_tester +from tests.ops.integration_tests.saas.connector_runner import ( + create_privacy_preference_history, +) + + +@register("opt_in_request_override", [SaaSRequestType.OPT_IN]) +def opt_in_request_override( + client: AuthenticatedClient, + policy: Policy, + privacy_request: PrivacyRequest, + secrets: Dict[str, Any], +) -> bool: + """A sample opt-in request override""" + return True + + +@register("opt_out_request_override", [SaaSRequestType.OPT_OUT]) +def opt_out_request_override( + client: AuthenticatedClient, + policy: Policy, + privacy_request: PrivacyRequest, + secrets: Dict[str, Any], +) -> bool: + """A sample opt-out request override""" + return True + + +class TestConsentRequestOverride: + @pytest.mark.parametrize( + "dsr_version, opt_in, expected_override_function_name, expected_saas_request_type", + [ + ("use_dsr_3_0", False, "opt_out_request_override", SaaSRequestType.OPT_OUT), + ("use_dsr_2_0", False, "opt_out_request_override", SaaSRequestType.OPT_OUT), + ("use_dsr_3_0", True, "opt_in_request_override", SaaSRequestType.OPT_IN), + ("use_dsr_2_0", True, "opt_in_request_override", SaaSRequestType.OPT_IN), + ], + ) + def test_old_consent_request( + self, + db, + consent_policy: Policy, + saas_consent_request_override_connection_config, + saas_consent_request_override_dataset_config, + privacy_request, + dsr_version, + request, + mocker, + opt_in, + expected_override_function_name, + expected_saas_request_type, + ): + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + saas_config = saas_consent_request_override_connection_config.get_saas_config() + dataset_name = saas_config.fides_key + + privacy_request.consent_preferences = [ + {"data_use": "marketing.advertising", "opt_in": opt_in} + ] + privacy_request.save(db) + + spy = mocker.spy(SaaSRequestOverrideFactory, "get_override") + + v = consent_runner_tester( + privacy_request, + consent_policy, + build_consent_dataset_graph([saas_consent_request_override_dataset_config]), + [saas_consent_request_override_connection_config], + {"email": "user@example.com"}, + db, + ) + assert v == {f"{dataset_name}:{dataset_name}": True} + spy.assert_called_once_with( + expected_override_function_name, expected_saas_request_type + ) + + @pytest.mark.parametrize( + "dsr_version, opt_in, expected_override_function_name, expected_saas_request_type", + [ + ("use_dsr_3_0", False, "opt_out_request_override", SaaSRequestType.OPT_OUT), + ("use_dsr_2_0", False, "opt_out_request_override", SaaSRequestType.OPT_OUT), + ("use_dsr_3_0", True, "opt_in_request_override", SaaSRequestType.OPT_IN), + ("use_dsr_2_0", True, "opt_in_request_override", SaaSRequestType.OPT_IN), + ], + ) + async def test_new_consent_request( + self, + db, + consent_policy, + saas_consent_request_override_connection_config, + saas_consent_request_override_dataset_config, + privacy_request, + dsr_version, + request, + mocker, + opt_in, + expected_override_function_name, + expected_saas_request_type, + ) -> None: + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + saas_config = saas_consent_request_override_connection_config.get_saas_config() + dataset_name = saas_config.fides_key + + identities = {"email": "user@example.com"} + privacy_request = PrivacyRequest( + id=f"test_consent_request_task_{random.randint(0, 1000)}", + status=PrivacyRequestStatus.pending, + policy_id=consent_policy.id, + ) + privacy_request.save(db) + + spy = mocker.spy(SaaSRequestOverrideFactory, "get_override") + + create_privacy_preference_history( + db, privacy_request, identities, opt_in=opt_in + ) + privacy_request.save(db) + v = consent_runner_tester( + privacy_request, + consent_policy, + build_consent_dataset_graph([saas_consent_request_override_dataset_config]), + [saas_consent_request_override_connection_config], + identities, + db, + ) + assert v == {f"{dataset_name}:{dataset_name}": True} + spy.assert_called_once_with( + expected_override_function_name, expected_saas_request_type + ) diff --git a/tests/ops/integration_tests/saas/request_override/test_mailchimp_override_task.py b/tests/ops/integration_tests/saas/request_override/test_mailchimp_override_task.py index 087dc48710..f1b2bfeac4 100644 --- a/tests/ops/integration_tests/saas/request_override/test_mailchimp_override_task.py +++ b/tests/ops/integration_tests/saas/request_override/test_mailchimp_override_task.py @@ -1,8 +1,22 @@ +from json import dumps +from typing import Any, Dict, List + +import pydash import pytest +from fides.api.graph.execution import ExecutionNode from fides.api.graph.graph import DatasetGraph +from fides.api.models.policy import Policy +from fides.api.models.privacy_request import PrivacyRequest from fides.api.schemas.redis_cache import Identity +from fides.api.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient +from fides.api.service.saas_request.saas_request_override_factory import ( + SaaSRequestType, + register, +) from fides.api.task.graph_task import get_cached_data_for_erasures +from fides.api.util.collection_util import Row from tests.conftest import access_runner_tester, erasure_runner_tester from tests.ops.graph.graph_test_util import assert_rows_match @@ -16,14 +30,105 @@ "custom" python functions. These sample functions perform identical behavior to what is performed by the standard framework for the standard Mailchimp config. -Therefore, our tests here are identical to the standard mailchimp test task, besides -that they reference the special "override" config and its associated dataset. +Therefore, our tests here are identical to the standard mailchimp test task, besides +that they reference the special "override" config and its associated dataset. With this, we verify that when the custom overrides are invoked by the "override" config, they execute successfully, which in this case happens to be the same behavior -as the standard Mailchimp config. +as the standard Mailchimp config. """ +@register("mailchimp_messages_access", [SaaSRequestType.READ]) +def mailchimp_messages_access( + client: AuthenticatedClient, + node: ExecutionNode, + policy: Policy, + privacy_request: PrivacyRequest, + input_data: Dict[str, List[Any]], + secrets: Dict[str, Any], +) -> List[Row]: + """ + Equivalent SaaS config for the code in this function. + + Request params still need to be defined for endpoints with overrides. + This is to provide the necessary reference and identity data as part + of graph traversal. The resulting values are passed in as parameters + so we don't need to define the data retrieval here. + + path: /3.0/conversations//messages + request_params: + - name: conversation_id + type: path + references: + - dataset: mailchimp_instance + field: conversations.id + direction: from + data_path: conversation_messages + postprocessors: + - strategy: filter + configuration: + field: from_email + value: + identity: email + """ + # gather request params + conversation_ids = input_data.get("conversation_id") + + # build and execute request for each input data value + processed_data = [] + if conversation_ids: + for conversation_id in conversation_ids: + response = client.send( + SaaSRequestParams( + method=HTTPMethod.GET, + path=f"/3.0/conversations/{conversation_id}/messages", + ) + ) + + # unwrap and post-process response + response_data = pydash.get(response.json(), "conversation_messages") + filtered_data = pydash.filter_( + response_data, + {"from_email": privacy_request.get_cached_identity_data().get("email")}, + ) + + # build up final result + processed_data.extend(filtered_data) + + return processed_data + + +@register("mailchimp_member_update", [SaaSRequestType.UPDATE]) +def mailchimp_member_update( + client: AuthenticatedClient, + param_values_per_row: List[Dict[str, Any]], + policy: Policy, + privacy_request: PrivacyRequest, + secrets: Dict[str, Any], +) -> int: + rows_updated = 0 + # each update_params dict correspond to a record that needs to be updated + for row_param_values in param_values_per_row: + # get params to be used in update request + list_id = row_param_values.get("list_id") + subscriber_hash = row_param_values.get("subscriber_hash") + + # in this case, we can just put the masked object fields object + # directly into the request body + update_body = dumps(row_param_values["masked_object_fields"]) + + client.send( + SaaSRequestParams( + method=HTTPMethod.PUT, + path=f"/3.0/lists/{list_id}/members/{subscriber_hash}", + body=update_body, + ) + ) + + rows_updated += 1 + return rows_updated + + @pytest.mark.integration_saas @pytest.mark.integration_saas_override @pytest.mark.asyncio diff --git a/tests/ops/service/saas_request/test_saas_request_override_factory.py b/tests/ops/service/saas_request/test_saas_request_override_factory.py index d4463dc265..0784bc0d07 100644 --- a/tests/ops/service/saas_request/test_saas_request_override_factory.py +++ b/tests/ops/service/saas_request/test_saas_request_override_factory.py @@ -74,6 +74,18 @@ def valid_update_override( pass +def valid_consent_override( + client: AuthenticatedClient, + policy: Policy, + privacy_request: PrivacyRequest, + secrets: Dict[str, Any], +) -> bool: + """ + A sample override function for consent requests with a valid function signature + """ + pass + + @pytest.mark.unit_saas class TestSaasRequestOverrideFactory: """ @@ -141,6 +153,36 @@ def test_register_update_and_delete(self): SaaSRequestOverrideFactory.get_override(f_id, SaaSRequestType.READ) assert f"Custom SaaS override '{f_id}' does not exist." in str(exc.value) + def test_register_opt_in_consent(self): + """ + Test registering a valid `opt_in` override function + """ + + f_id = uuid() + register(f_id, SaaSRequestType.OPT_IN)(valid_consent_override) + assert valid_consent_override == SaaSRequestOverrideFactory.get_override( + f_id, SaaSRequestType.OPT_IN + ) + + with pytest.raises(NoSuchSaaSRequestOverrideException) as exc: + SaaSRequestOverrideFactory.get_override(f_id, SaaSRequestType.READ) + assert f"Custom SaaS override '{f_id}' does not exist." in str(exc.value) + + def test_register_opt_out_consent(self): + """ + Test registering a valid `opt_out` override function + """ + + f_id = uuid() + register(f_id, SaaSRequestType.OPT_OUT)(valid_consent_override) + assert valid_consent_override == SaaSRequestOverrideFactory.get_override( + f_id, SaaSRequestType.OPT_OUT + ) + + with pytest.raises(NoSuchSaaSRequestOverrideException) as exc: + SaaSRequestOverrideFactory.get_override(f_id, SaaSRequestType.READ) + assert f"Custom SaaS override '{f_id}' does not exist." in str(exc.value) + def test_reregister_override(self): """ Test that registering a new override with the same ID and same request type