Adding privacy_request_id placeholder (#911)

CHANGELOG.md

@@ -54,6 +54,7 @@ The types of changes are:
 * Erasure support for Zendesk [#775](https://github.com/ethyca/fidesops/pull/775)
 * Adds endpoint to get the secrets required for different connectors [#795](https://github.com/ethyca/fidesops/pull/795)
 * Adds Vault for secrets management [#688](https://github.com/ethyca/fidesops/pull/869)
+* Adds privacy_request_id placeholder to use in SaaS configs [#911](https://github.com/ethyca/fidesops/pull/911)
 
 ### Changed
 

data/saas/config/saas_example_config.yml

@@ -183,3 +183,21 @@ saas_config:
             {
               <all_object_fields>
             }
+    - name: data_management
+      requests:
+        read:
+          method: GET
+          path: /v1/privacy_request/<privacy_request_id>
+          param_values:
+            - name: placeholder
+              identity: email
+        update:
+          method: POST
+          path: /v1/privacy_request/
+          param_values:
+            - name: placeholder
+              identity: email
+          body: |
+            {
+              "unique_id": "<privacy_request_id>"
+            }

data/saas/dataset/saas_example_dataset.yml

@@ -90,15 +90,18 @@ dataset:
                     fidesops_meta:
                       data_type: string
                   - name: zip
-                    data_categories: [user.provided.identifiable.contact.postal_code]
+                    data_categories:
+                      [user.provided.identifiable.contact.postal_code]
                     fidesops_meta:
                       data_type: string
                   - name: country
-                    data_categories: [user.provided.identifiable.contact.country]
+                    data_categories:
+                      [user.provided.identifiable.contact.country]
                     fidesops_meta:
                       data_type: string
               - name: PHONE
-                data_categories: [user.provided.identifiable.contact.phone_number]
+                data_categories:
+                  [user.provided.identifiable.contact.phone_number]
                 fidesops_meta:
                   data_type: string
               - name: BIRTHDAY
@@ -154,9 +157,9 @@ dataset:
       - name: projects
         fields:
           - name: id
-            data_categories: [ system.operations ]
+            data_categories: [system.operations]
           - name: slug
-            data_categories: [ system.operations ]
+            data_categories: [system.operations]
           - name: organization
             fields:
               - name: slug
@@ -165,21 +168,21 @@ dataset:
       - name: users
         fields:
           - name: id
-            data_categories: [ system.operations ]
+            data_categories: [system.operations]
           - name: name
-            data_categories: [ user.provided.identifiable.name ]
+            data_categories: [user.provided.identifiable.name]
           - name: user
             fields:
               - name: username
-                data_categories: [ user.provided.identifiable.name ]
+                data_categories: [user.provided.identifiable.name]
               - name: email
-                data_categories: [ user.provided.identifiable.contact.email ]
+                data_categories: [user.provided.identifiable.contact.email]
               - name: name
-                data_categories: [ user.provided.identifiable.name ]
+                data_categories: [user.provided.identifiable.name]
               - name: ipAddress
-                data_categories: [ user.derived.identifiable.device.ip_address ]
+                data_categories: [user.derived.identifiable.device.ip_address]
           - name: email
-            data_categories: [ user.provided.identifiable.contact.email ]
+            data_categories: [user.provided.identifiable.contact.email]
       - name: customer
         fields:
           - name: id
@@ -200,4 +203,10 @@ dataset:
           - name: created
             data_categories: [system.operations]
             fidesops_meta:
-              read_only: True
+              read_only: True
+      - name: data_management
+        fields:
+          - name: id
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string

src/fidesops/service/connectors/saas_connector.py

@@ -40,6 +40,7 @@ def __init__(self, configuration: ConnectionConfig):
         self.client_config = self.saas_config.client_config  # type: ignore
         self.endpoints = self.saas_config.top_level_endpoint_dict  # type: ignore
         self.collection_name: Optional[str] = None
+        self.privacy_request: Optional[PrivacyRequest] = None
 
     def query_config(self, node: TraversalNode) -> SaaSQueryConfig:
         """
@@ -49,7 +50,11 @@ def query_config(self, node: TraversalNode) -> SaaSQueryConfig:
         # store collection_name for logging purposes
         self.collection_name = node.address.collection
         return SaaSQueryConfig(
-            node, self.endpoints, self.secrets, self.saas_config.data_protection_request  # type: ignore
+            node,
+            self.endpoints,
+            self.secrets,  # type: ignore
+            self.saas_config.data_protection_request,  # type: ignore
+            self.privacy_request,  # type: ignore
         )
 
     def test_connection(self) -> Optional[ConnectionTestStatus]:
@@ -103,7 +108,9 @@ def retrieve_data(
         input_data: Dict[str, List[Any]],
     ) -> List[Row]:
         """Retrieve data from SaaS APIs"""
+
         # generate initial set of requests if read request is defined, otherwise raise an exception
+        self.privacy_request = privacy_request
         query_config: SaaSQueryConfig = self.query_config(node)
         read_request: Optional[SaaSRequest] = query_config.get_request_by_action("read")
         if not read_request:
@@ -139,6 +146,7 @@ def execute_prepared_request(
         Executes the prepared request and handles response postprocessing and pagination.
         Returns processed data and request_params for next page of data if available.
         """
+
         client: AuthenticatedClient = self.create_client_from_request(saas_request)
         response: Response = client.send(prepared_request, saas_request.ignore_errors)
         response = self._handle_errored_response(saas_request, response)
@@ -186,6 +194,7 @@ def process_response_data(
 
         The final result is returned as a list of processed objects.
         """
+
         rows: List[Row] = []
         processed_data = response_data
         for postprocessor in postprocessors or []:
@@ -230,6 +239,7 @@ def mask_data(
     ) -> int:
         """Execute a masking request. Return the number of rows that have been updated."""
 
+        self.privacy_request = privacy_request
         query_config = self.query_config(node)
         masking_request = query_config.get_masking_request()
         if not masking_request:

src/fidesops/service/connectors/saas_query_config.py

@@ -33,12 +33,14 @@ def __init__(
         endpoints: Dict[str, Endpoint],
         secrets: Dict[str, Any],
         data_protection_request: Optional[SaaSRequest] = None,
+        privacy_request: Optional[PrivacyRequest] = None,
     ):
         super().__init__(node)
         self.collection_name = node.address.collection
         self.endpoints = endpoints
         self.secrets = secrets
         self.data_protection_request = data_protection_request
+        self.privacy_request = privacy_request
         self.action: Optional[str] = None
 
     def get_request_by_action(self, action: str) -> Optional[SaaSRequest]:
@@ -152,6 +154,9 @@ def generate_query(
                     self.secrets, param_value.connector_param
                 )
 
+        if self.privacy_request:
+            param_values["privacy_request_id"] = self.privacy_request.id
+
         # map param values to placeholders in path, headers, and query params
         saas_request_params: SaaSRequestParams = saas_util.map_param_values(
             self.action, self.collection_name, current_request, param_values  # type: ignore
@@ -192,6 +197,9 @@ def generate_update_stmt(  # pylint: disable=R0914
                     self.secrets, param_value.connector_param
                 )
 
+        if self.privacy_request:
+            param_values["privacy_request_id"] = self.privacy_request.id
+
         # remove any row values for fields marked as read-only, these will be omitted from all update maps
         for field_path, field in self.field_map().items():
             if field.read_only:

tests/api/v1/endpoints/test_saas_config_endpoints.py

@@ -244,7 +244,7 @@ def test_patch_saas_config_update(
         )
         saas_config = connection_config.saas_config
         assert saas_config is not None
-        assert len(saas_config["endpoints"]) == 6
+        assert len(saas_config["endpoints"]) == 7
 
 
 def get_saas_config_url(connection_config: Optional[ConnectionConfig] = None) -> str:
@@ -318,7 +318,7 @@ def test_get_saas_config(
             response_body["fides_key"]
             == saas_example_connection_config.get_saas_config().fides_key
         )
-        assert len(response_body["endpoints"]) == 7
+        assert len(response_body["endpoints"]) == 8
         assert response_body["type"] == "custom"
 
 

tests/service/connectors/test_queryconfig.py

@@ -636,6 +636,10 @@ def test_generate_query(
             CollectionAddress(saas_config.fides_key, "payment_methods")
         ]
 
+        data_management = combined_traversal.traversal_node_dict[
+            CollectionAddress(saas_config.fides_key, "data_management")
+        ]
+
         # static path with single query param
         config = SaaSQueryConfig(member, endpoints, {})
         prepared_request: SaaSRequestParams = config.generate_query(
@@ -700,6 +704,14 @@ def test_generate_query(
             "query": "[email protected]",
         }
 
+        # using privacy_request_id placeholder
+        config = SaaSQueryConfig(data_management, endpoints, {}, None, privacy_request)
+        prepared_request = config.generate_query(
+            {"email": ["[email protected]"]}, policy
+        )
+        assert prepared_request.method == HTTPMethod.GET.value
+        assert prepared_request.path == f"/v1/privacy_request/{privacy_request.id}"
+
     def test_generate_update_stmt(
         self,
         erasure_policy_string_rewrite,
@@ -714,6 +726,10 @@ def test_generate_update_stmt(
             CollectionAddress(saas_config.fides_key, "member")
         ]
 
+        data_management = combined_traversal.traversal_node_dict[
+            CollectionAddress(saas_config.fides_key, "data_management")
+        ]
+
         config = SaaSQueryConfig(member, endpoints, {}, update_request)
         row = {
             "id": "123",
@@ -730,10 +746,18 @@ def test_generate_update_stmt(
         assert prepared_request.path == "/3.0/lists/abc/members/123"
         assert prepared_request.headers == {"Content-Type": "application/json"}
         assert prepared_request.query_params == {}
-        assert (
-            prepared_request.body
-            == '{\n  "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}\n}\n'
+        assert json.loads(prepared_request.body) == {
+            "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}
+        }
+
+        # using privacy_request_id placeholder
+        config = SaaSQueryConfig(data_management, endpoints, {}, None, privacy_request)
+        prepared_request = config.generate_update_stmt(
+            row, erasure_policy_string_rewrite, privacy_request
         )
+        assert prepared_request.method == HTTPMethod.POST.value
+        assert prepared_request.path == "/v1/privacy_request/"
+        assert json.loads(prepared_request.body) == {"unique_id": privacy_request.id}
 
     def test_generate_update_stmt_custom_http_method(
         self,
@@ -768,10 +792,9 @@ def test_generate_update_stmt_custom_http_method(
         assert prepared_request.path == "/3.0/lists/abc/members/123"
         assert prepared_request.headers == {"Content-Type": "application/json"}
         assert prepared_request.query_params == {}
-        assert (
-            prepared_request.body
-            == '{\n  "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}\n}\n'
-        )
+        assert json.loads(prepared_request.body) == {
+            "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}
+        }
 
     def test_generate_update_stmt_with_request_body(
         self,
@@ -847,7 +870,7 @@ def test_generate_update_stmt_with_request_body(
         assert prepared_request.path == "/2.0/payment_methods"
         assert prepared_request.headers == {"Content-Type": "application/json"}
         assert prepared_request.query_params == {}
-        assert prepared_request.body == '{\n  "customer_name": "MASKED"\n}\n'
+        assert json.loads(prepared_request.body) == {"customer_name": "MASKED"}
 
     def test_generate_update_stmt_with_url_encoded_body(
         self,