Multi-value support for custom privacy request fields

clients/admin-ui/src/features/privacy-requests/ApprovePrivacyRequestModal.tsx

@@ -98,7 +98,10 @@ const ApprovePrivacyRequestModal = ({
                         fontSize="sm"
                         mr={2}
                       >
-                        {item.value} (Unverified)
+                        {Array.isArray(item.value)
+                          ? item.value.join(", ")
+                          : item.value}{" "}
+                        (Unverified)
                       </Text>
                     </Flex>
                   </ListItem>

clients/admin-ui/src/features/privacy-requests/SubjectIdentities.tsx

@@ -102,7 +102,14 @@ const SubjectIdentities = ({ subjectRequest }: SubjectIdentitiesProps) => {
                     {item.label}:
                   </Text>
                   <Text color="gray.600" fontWeight="500" fontSize="sm" mr={2}>
-                    <PII data={item.value} revealPII={revealPII} />
+                    <PII
+                      data={
+                        Array.isArray(item.value)
+                          ? item.value.join(", ")
+                          : item.value
+                      }
+                      revealPII={revealPII}
+                    />
                   </Text>
                   <Tag
                     color="white"

src/fides/api/models/privacy_request.py

@@ -34,7 +34,7 @@
     NoCachedManualWebhookEntry,
     PrivacyRequestPaused,
 )
-from fides.api.cryptography.cryptographic_util import generate_salt, hash_with_salt
+from fides.api.cryptography.cryptographic_util import hash_with_salt
 from fides.api.db.base_class import Base  # type: ignore[attr-defined]
 from fides.api.db.base_class import JSONTypeOverride
 from fides.api.db.util import EnumColumn
@@ -60,9 +60,14 @@
 from fides.api.schemas.redis_cache import (
     CustomPrivacyRequestField as CustomPrivacyRequestFieldSchema,
 )
-from fides.api.schemas.redis_cache import Identity, IdentityBase
+from fides.api.schemas.redis_cache import (
+    CustomPrivacyRequestFieldValue,
+    Identity,
+    IdentityBase,
+)
 from fides.api.tasks import celery_app
 from fides.api.util.cache import (
+    CustomJSONEncoder,
     FidesopsRedis,
     get_all_cache_keys_for_privacy_request,
     get_async_task_tracking_cache_key,
@@ -364,7 +369,7 @@ def cache_custom_privacy_request_fields(
                 if item is not None:
                     cache.set_with_autoexpire(
                         get_custom_privacy_request_field_cache_key(self.id, key),
-                        item.value,
+                        json.dumps(item.value, cls=CustomJSONEncoder),
                     )
         else:
             logger.info(
@@ -533,7 +538,12 @@ def get_cached_custom_privacy_request_fields(self) -> Dict[str, Any]:
         prefix = f"id-{self.id}-custom-privacy-request-field-*"
         cache: FidesopsRedis = get_cache()
         keys = cache.keys(prefix)
-        return {key.split("-")[-1]: cache.get(key) for key in keys}
+        result = {}
+        for key in keys:
+            value = cache.get(key)
+            if value:
+                result[key.split("-")[-1]] = json.loads(value)
+        return result
 
     def get_results(self) -> Dict[str, Any]:
         """Retrieves all cached identity data associated with this Privacy Request"""
@@ -1146,16 +1156,23 @@ def __tablename__(self) -> str:
     @classmethod
     def hash_value(
         cls,
-        value: str,
+        value: CustomPrivacyRequestFieldValue,
         encoding: str = "UTF-8",
-    ) -> str:
-        """Utility function to hash the value with a generated salt"""
-        salt = generate_salt()
-        hashed_value = hash_with_salt(
-            value.encode(encoding),
-            salt.encode(encoding),
-        )
-        return hashed_value
+    ) -> Union[str, List[str]]:
+        """Utility function to hash the value(s) with a generated salt"""
+
+        def hash_single_value(value: Union[str, int]) -> str:
+            SALT = "$2b$12$UErimNtlsE6qgYf2BrI1Du"
+            value_str = str(value)
+            hashed_value = hash_with_salt(
+                value_str.encode(encoding),
+                SALT.encode(encoding),
+            )
+            return hashed_value
+
+        if isinstance(value, list):
+            return [hash_single_value(item) for item in value]
+        return hash_single_value(value)
 
 
 class Consent(Base):

src/fides/api/schemas/redis_cache.py

@@ -1,7 +1,7 @@
 import uuid
-from typing import Any, Optional
+from typing import List, Optional, Union
 
-from pydantic import EmailStr, Extra, validator
+from pydantic import EmailStr, Extra, StrictInt, StrictStr, validator
 
 from fides.api.custom_types import PhoneNumber
 from fides.api.schemas.base_class import FidesSchema
@@ -44,8 +44,14 @@ def validate_fides_user_device_id(cls, v: Optional[str]) -> Optional[str]:
         return v
 
 
+CustomPrivacyRequestFieldValue = Union[
+    Union[StrictInt, StrictStr], List[Union[StrictInt, StrictStr]]
+]
+
+
 class CustomPrivacyRequestField(FidesSchema):
     """Schema for custom privacy request fields."""
 
     label: str
-    value: Any
+    # use StrictInt and StrictStr to avoid type coercion and maintain the original types
+    value: CustomPrivacyRequestFieldValue

src/fides/api/util/saas_util.py

@@ -349,7 +349,8 @@ def assign_placeholders(value: Any, param_values: Dict[str, Any]) -> Optional[An
             # removes outer {} wrapper from body for greater flexibility in custom body config
             if isinstance(placeholder_value, dict):
                 placeholder_value = json.dumps(placeholder_value)[1:-1]
-
+            if isinstance(placeholder_value, list):
+                placeholder_value = json.dumps(placeholder_value)
             if placeholder_value is not None:
                 value = value.replace(f"<{full_placeholder}>", str(placeholder_value))
             elif is_optional:

tests/fixtures/saas/test_data/saas_custom_privacy_request_fields_config.yml

@@ -0,0 +1,53 @@
+saas_config:
+  fides_key: <instance_fides_key>
+  name: Custom Privacy Request Fields
+  type: custom_privacy_request_fields
+  description: A sample schema to test custom privacy request fields
+  version: 0.0.1
+
+  connector_params:
+    - name: <api_key>
+
+  client_config:
+    protocol: https
+    host: localhost
+    authentication:
+      strategy: bearer
+      configuration:
+        token: <api_key>
+
+  test_request:
+    method: GET
+    path: /ping
+
+  endpoints:
+    - name: user
+      requests:
+        read:
+          method: POST
+          path: /v1/user-search
+          param_values:
+            - name: placeholder
+              identity: email
+          query_params:
+            - name: first_name
+              value: <custom_privacy_request_fields.first_name>
+          body: |
+            {
+              "last_name": "<custom_privacy_request_fields.last_name>",
+              "order_id": "<custom_privacy_request_fields.order_id?>",
+              "subscriber_ids": <custom_privacy_request_fields.subscriber_ids>,
+              "account_ids": <custom_privacy_request_fields.account_ids>,
+              "support_id": <custom_privacy_request_fields.support_id>
+            }
+          ignore_errors: True
+        update:
+          method: PUT
+          path: /v1/user
+          body: |
+            {
+              "user_info": {
+                <custom_privacy_request_fields>
+              }
+            }
+          ignore_errors: True

tests/fixtures/saas/test_data/saas_custom_privacy_request_fields_dataset.yml

@@ -0,0 +1,12 @@
+dataset:
+  - fides_key: <instance_fides_key>
+    name: Custom Privacy Request Fields Dataset
+    description: A sample dataset for testing
+    collections:
+      - name: user
+        fields:
+          - name: id
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
+              primary_key: True

tests/fixtures/saas/test_data/saas_example_config.yml

@@ -319,7 +319,9 @@ saas_config:
           body: |
             {
               "last_name": "<custom_privacy_request_fields.last_name>",
-              "order_id": "<custom_privacy_request_fields.order_id?>"
+              "order_id": "<custom_privacy_request_fields.order_id?>",
+              "subscriber_ids": <custom_privacy_request_fields.subscriber_ids>,
+              "account_ids": <custom_privacy_request_fields.account_ids>
             }
         update:
           method: POST
@@ -342,4 +344,3 @@ saas_config:
                 - dataset: saas_connector_example
                   field: users.list_ids
                   direction: from
-

tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py

@@ -215,6 +215,48 @@ def test_create_privacy_request_stores_custom_fields(
         pr.delete(db=db)
         assert run_access_request_mock.called
 
+    @mock.patch(
+        "fides.api.service.privacy_request.request_runner_service.run_privacy_request.delay"
+    )
+    def test_create_privacy_request_stores_multivalue_custom_fields(
+        self,
+        run_access_request_mock,
+        url,
+        db,
+        api_client: TestClient,
+        policy,
+        allow_custom_privacy_request_field_collection_enabled,
+    ):
+        TEST_EMAIL = "[email protected]"
+        TEST_CUSTOM_FIELDS = {
+            "first_name": {"label": "First name", "value": "John"},
+            "subscriber_ids": {"label": "Subscriber IDs", "value": ["123", "456"]},
+            "account_ids": {"label": "Account IDs", "value": [123, 456]},
+        }
+        data = [
+            {
+                "requested_at": "2021-08-30T16:09:37.359Z",
+                "policy_key": policy.key,
+                "identity": {
+                    "email": TEST_EMAIL,
+                },
+                "custom_privacy_request_fields": TEST_CUSTOM_FIELDS,
+            }
+        ]
+        resp = api_client.post(url, json=data)
+        assert resp.status_code == 200
+        response_data = resp.json()["succeeded"]
+        assert len(response_data) == 1
+        pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"])
+        persisted_identity = pr.get_persisted_identity()
+        assert persisted_identity.email == TEST_EMAIL
+        persisted_custom_privacy_request_fields = (
+            pr.get_persisted_custom_privacy_request_fields()
+        )
+        assert persisted_custom_privacy_request_fields == TEST_CUSTOM_FIELDS
+        pr.delete(db=db)
+        assert run_access_request_mock.called
+
     @mock.patch(
         "fides.api.service.privacy_request.request_runner_service.run_privacy_request.delay"
     )

tests/ops/integration_tests/saas/test_oracle_responsys_task.py

@@ -5,6 +5,7 @@
 
 
 @pytest.mark.skip("Enterprise account only")
+@pytest.mark.integration_saas
 class TestOracleResponsysConnector:
     def test_connection(self, oracle_responsys_runner: ConnectorRunner):
         oracle_responsys_runner.test_connection()