CON-206: Responsys access and erasure

CHANGELOG.md

@@ -19,6 +19,7 @@ The types of changes are:
 
 ### Added
 - Add Great Britain as a consent option [#4628](https://github.com/ethyca/fides/pull/4628)
+- Access and Erasure support for Oracle Responsys [#4618](https://github.com/ethyca/fides/pull/4628)
 
 ## [2.30.1](https://github.com/ethyca/fides/compare/2.30.0...2.30.1)
 

data/saas/config/oracle_responsys_config.yml

@@ -0,0 +1,77 @@
+saas_config:
+  fides_key: <instance_fides_key>
+  name: Oracle Responsys
+  type: oracle_responsys
+  description: A sample schema representing the Oracle Responsys connector for Fides
+  version: 0.0.1
+
+  connector_params:
+    - name: domain
+      description: Your Oracle Responsys domain
+    - name: username
+      description: Your Oracle Responsys username
+    - name: password
+      description: Your Oracle Responsys password
+      sensitive: True
+
+  client_config:
+    protocol: https
+    host: <domain>
+    authentication:
+      strategy: oracle_responsys
+      configuration:
+        username: <username>
+        password: <password>
+
+  test_request:
+    method: GET
+    path: /rest/api/v1.3/lists
+
+  endpoints:
+    - name: profile_list
+      requests:
+        read:
+          - method: GET
+            path: /rest/api/v1.3/lists
+            param_values:
+              - name: placeholder_email
+                identity: email
+          - method: GET
+            path: /rest/api/v1.3/lists
+            param_values:
+              - name: placeholder_phone
+                identity: phone_number
+    - name: profile_list_recipient
+      requests:
+        read:
+          - request_override: oracle_responsys_profile_list_recipients_read
+            param_values:
+              - name: profile_list_id
+                references:
+                  - dataset: <instance_fides_key>
+                    field: profile_list.name
+                    direction: from
+              - name: email
+                identity: email
+          - request_override: oracle_responsys_profile_list_recipients_read
+            param_values:
+              - name: profile_list_id
+                references:
+                  - dataset: <instance_fides_key>
+                    field: profile_list.name
+                    direction: from
+              - name: phone_number
+                identity: phone_number
+        delete:
+          request_override: oracle_responsys_profile_list_recipients_delete
+          param_values:
+            - name: profile_list_id
+              references:
+                - dataset: <instance_fides_key>
+                  field: profile_list_recipient.profile_list_id
+                  direction: from
+            - name: responsys_id
+              references:
+                - dataset: <instance_fides_key>
+                  field: profile_list_recipient.riid
+                  direction: from

data/saas/dataset/oracle_responsys_dataset.yml

@@ -0,0 +1,112 @@
+dataset:
+  - fides_key: <instance_fides_key>
+    name: oracle_responsys
+    description: A sample dataset representing the Oracle Responsys connector for Fides
+    collections:
+      - name: profile_list_recipient
+        fields:
+          - name: profile_list_id
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: riid
+            data_categories: [user.unique_id]
+            fidesops_meta:
+              primary_key: True
+              data_type: string
+          - name: customer_id
+            data_categories: [user.unique_id]
+            fidesops_meta:
+              data_type: string
+          - name: email_address
+            data_categories: [user.contact.email]
+            fidesops_meta:
+              data_type: string
+          - name: email_domain
+            data_categories: [user.contact.email]
+            fidesops_meta:
+              data_type: string
+          - name: email_isp
+            data_categories: [user.contact.email]
+            fidesops_meta:
+              data_type: string
+          - name: email_format
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: email_permission_status
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: email_deliverability_status
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: email_permission_reason
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: email_md5_hash
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: email_sha256_hash
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: mobile_number
+            data_categories: [user.contact.phone_number]
+            fidesops_meta:
+              data_type: string
+          - name: mobile_country
+            data_categories: [user.contact.address.country]
+            fidesops_meta:
+              data_type: string
+          - name: mobile_permission_status
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: mobile_permission_reason
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: mobile_deliverability_status
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: postal_street_1
+            data_categories: [user.contact.address.street]
+            fidesops_meta:
+              data_type: string
+          - name: postal_street_2
+            data_categories: [user.contact.address.street]
+            fidesops_meta:
+              data_type: string
+          - name: city
+            data_categories: [user.contact.address.city]
+            fidesops_meta:
+              data_type: string
+          - name: state
+            data_categories: [user.contact.address.state]
+            fidesops_meta:
+              data_type: string
+          - name: postal_code
+            data_categories: [user.contact.address.postal_code]
+            fidesops_meta:
+              data_type: string
+          - name: country
+            data_categories: [user.contact.address.country]
+            fidesops_meta:
+              data_type: string
+          - name: postal_permission_status
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: postal_permission_reason
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: postal_deliverability_status
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string

src/fides/api/service/saas_request/override_implementations/authentication_strategy_oracle_responsys.py

@@ -0,0 +1,60 @@
+from typing import Dict, Optional, cast
+
+from requests import PreparedRequest, post
+
+from fides.api.common_exceptions import FidesopsException
+from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.schemas.saas.strategy_configuration import StrategyConfiguration
+from fides.api.service.authentication.authentication_strategy import (
+    AuthenticationStrategy,
+)
+from fides.api.util.saas_util import assign_placeholders
+
+
+class OracleResponsysAuthenticationConfiguration(StrategyConfiguration):
+    """
+    Parameters to authorize a Oracle Responsys connection
+    """
+
+    username: str
+    password: str
+
+
+class OracleResponsysAuthenticationStrategy(AuthenticationStrategy):
+    """
+    Generates a token from the provided key and secret.
+    """
+
+    name = "oracle_responsys"
+    configuration_model = OracleResponsysAuthenticationConfiguration
+
+    def __init__(self, configuration: OracleResponsysAuthenticationConfiguration):
+        self.username = configuration.username
+        self.password = configuration.password
+
+    def add_authentication(
+        self, request: PreparedRequest, connection_config: ConnectionConfig
+    ) -> PreparedRequest:
+        """
+        Retrieves a token using the provided username and password
+        """
+
+        secrets = cast(Dict, connection_config.secrets)
+        domain: Optional[str] = secrets.get("domain")
+        username: Optional[str] = assign_placeholders(self.username, secrets)
+        password: Optional[str] = assign_placeholders(self.password, secrets)
+
+        response = post(
+            url=f"https://{domain}/rest/api/v1.3/auth/token",
+            headers={"Content-Type": "application/x-www-form-urlencoded"},
+            data={"user_name": username, "password": password, "auth_type": "password"},
+        )
+
+        if response.ok:
+            json_response = response.json()
+            token = json_response.get("authToken")
+        else:
+            raise FidesopsException(f"Unable to get token {response.json()}")
+
+        request.headers["Authorization"] = token
+        return request

src/fides/api/service/saas_request/override_implementations/oracle_responsys_request_overrides.py

@@ -0,0 +1,125 @@
+from typing import Any, Dict, List
+
+import json
+import pydash
+
+from fides.api.common_exceptions import FidesopsException
+from fides.api.graph.traversal import TraversalNode
+from fides.api.models.policy import Policy
+from fides.api.models.privacy_request import PrivacyRequest
+from fides.api.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams
+from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient
+from fides.api.service.saas_request.saas_request_override_factory import (
+    SaaSRequestType,
+    register,
+)
+from fides.api.util.collection_util import Row
+from fides.api.util.saas_util import get_identity
+
+
+@register("oracle_responsys_profile_list_recipients_read", [SaaSRequestType.READ])
+def oracle_responsys_profile_list_recipients_read(
+    client: AuthenticatedClient,
+    node: TraversalNode,
+    policy: Policy,
+    privacy_request: PrivacyRequest,
+    input_data: Dict[str, List[Any]],
+    secrets: Dict[str, Any],
+) -> List[Row]:
+    """
+    Retrieve data from each profile list.
+
+    The members endpoint returns data in two separate arrays: one for the keys and one for the values for each result.
+    {
+    "recordData": {
+        "fieldNames": [
+        <list of field names>
+        ],
+        "records": [
+            [
+                <list of field values, corresponding to the fieldNames>
+            ]
+        ]
+    }
+    """
+    list_ids = input_data.get("profile_list_id", [])
+    results = []
+
+    identity = get_identity(privacy_request)
+    if identity == "email":
+        query_ids = input_data.get("email", [])
+        query_attribute = "e"
+    elif identity == "phone_number":
+        query_ids = input_data.get("phone_number", [])
+        # Oracle Responsys doesn't support the initial + in phone numbers
+        for idx, query_id in enumerate(query_ids):
+            query_ids[idx] = query_id[1:] if query_id[0] == "+" else query_id
+        query_attribute = "m"
+    else:
+        raise FidesopsException(
+            "Unsupported identity type for Oracle Responsys connector. Currently only `email` and `phone_number` are supported"
+        )
+
+    body = {"fieldList": ["all"], "ids": query_ids, "queryAttribute": query_attribute}
+    for list_id in list_ids:
+        members_response = client.send(
+            SaaSRequestParams(
+                method=HTTPMethod.POST,
+                path=f"/rest/api/v1.3/lists/{list_id}/members",
+                query_params={"action": "get"},
+                body=json.dumps(body),
+                headers={"Content-Type": "application/json"},
+            ),
+            [404],  # Returns a 404 if no list member is found
+        )
+        response_data = pydash.get(members_response.json(), "recordData")
+        if response_data:
+            normalized_field_names = [
+                field.lower().rstrip("_") for field in response_data["fieldNames"]
+            ]
+            serialized_data = [
+                dict(zip(normalized_field_names, records))
+                for records in response_data["records"]
+            ]
+
+            for record in serialized_data:
+                # Filter out the keys with falsy values and append it
+                filtered_records = {
+                    key: value for key, value in record.items() if value
+                }
+                filtered_records["profile_list_id"] = list_id
+                results.append(filtered_records)
+
+    return results
+
+
+@register("oracle_responsys_profile_list_recipients_delete", [SaaSRequestType.DELETE])
+def oracle_responsys_profile_list_recipients_delete(
+    client: AuthenticatedClient,
+    param_values_per_row: List[Dict[str, Any]],
+    policy: Policy,
+    privacy_request: PrivacyRequest,
+    secrets: Dict[str, Any],
+) -> int:
+    """
+    Deletes data from each profile list. Upon deletion from the list, PET data is also deleted.
+    """
+    rows_deleted = 0
+    # each delete_params dict correspond to a record that needs to be deleted
+    for row_param_values in param_values_per_row:
+        # get params to be used in delete request
+        all_object_fields = row_param_values["all_object_fields"]
+
+        list_id = all_object_fields["profile_list_id"]
+        responsys_id = row_param_values.get("responsys_id")
+
+        if responsys_id:
+            client.send(
+                SaaSRequestParams(
+                    method=HTTPMethod.DELETE,
+                    path=f"/rest/api/v1.3/lists/{list_id}/members/{responsys_id}",
+                )
+            )
+
+            rows_deleted += 1
+    return rows_deleted