Add further testing for code cov (a lot of this is tested multiple ti…

…mes in saas connector tests as well).

tests/fixtures/saas/klaviyo_fixtures.py

@@ -28,7 +28,7 @@ def klaviyo_secrets(saas_config):
     return {
         "domain": pydash.get(saas_config, "klaviyo.domain") or secrets["domain"],
         "api_key": pydash.get(saas_config, "klaviyo.api_key") or secrets["api_key"],
-        "revision": pydash.get(saas_config, "klaviyo.revision") or secrets["revision"]
+        "revision": pydash.get(saas_config, "klaviyo.revision") or secrets["revision"],
     }
 
 
@@ -107,28 +107,28 @@ def klaviyo_dataset_config(
     dataset.delete(db=db)
     ctl_dataset.delete(db=db)
 
+
 @pytest.fixture(scope="function")
 def klaviyo_create_erasure_data(
     klaviyo_connection_config: ConnectionConfig, klaviyo_erasure_identity_email: str
 ) -> None:
-
     klaviyo_secrets = klaviyo_connection_config.secrets
     base_url = f"https://{klaviyo_secrets['domain']}"
     headers = {
         "Authorization": f"Klaviyo-API-Key {klaviyo_secrets['api_key']}",
-        "revision": klaviyo_secrets['revision']
+        "revision": klaviyo_secrets["revision"],
     }
     # user
     body = {
         "data": {
             "type": "profile",
-            "attributes": {
-                "email": klaviyo_erasure_identity_email
-            }
+            "attributes": {"email": klaviyo_erasure_identity_email},
         }
     }
 
-    users_response = requests.post(url=f"{base_url}/api/profiles/", headers=headers, json=body)
+    users_response = requests.post(
+        url=f"{base_url}/api/profiles/", headers=headers, json=body
+    )
     user = users_response.json()
 
     yield user

tests/ops/api/v1/test_utils.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+
+import pytest
+from fastapi import HTTPException
+
+from fides.api.ops.api.v1.endpoints.utils import validate_start_and_end_filters
+
+
+class TestValidateStartAndEndFilters:
+    def test_start_date_after_end_date(self):
+        with pytest.raises(HTTPException):
+            validate_start_and_end_filters(
+                [(datetime(2020, 1, 1), datetime(2020, 2, 2), "created")]
+            )
+
+    def test_value_is_none(self):
+        validate_start_and_end_filters([(None, datetime(2020, 2, 2), "created")])
+        validate_start_and_end_filters([(datetime(2020, 1, 1), None, "created")])
+
+    def test_value_is_not_a_date(self):
+        validate_start_and_end_filters([(1, datetime(2020, 2, 2), "created")])
+        validate_start_and_end_filters([(datetime(2020, 1, 1), 1, "created")])
+
+    def test_valid_dates(self):
+        validate_start_and_end_filters(
+            [(datetime(2020, 2, 2), datetime(2020, 2, 1), "created")]
+        )

tests/ops/integration_tests/saas/test_google_analytics_task.py

@@ -322,6 +322,7 @@ async def test_google_analytics_consent_request_task_new_workflow(
 @pytest.mark.integration_saas
 @pytest.mark.integration_google_analytics
 @pytest.mark.asyncio
+@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors")
 @mock.patch("fides.api.ops.service.connectors.saas_connector.AuthenticatedClient.send")
 async def test_google_analytics_consent_request_task_new_errored_workflow(
     mocked_client_send,

tests/ops/integration_tests/saas/test_klaviyo_task.py

@@ -1,4 +1,5 @@
 import random
+
 import pytest
 import requests
 
@@ -54,17 +55,14 @@ async def test_klaviyo_access_request_task(
     assert_rows_match(
         v[f"{dataset_name}:profiles"],
         min_size=1,
-        keys=[
-            "type",
-            "id",
-            "attributes",
-            "links",
-            "relationships"
-        ],
+        keys=["type", "id", "attributes", "links", "relationships"],
     )
 
     # verify we only returned data for our identity email
-    assert v[f"{dataset_name}:profiles"][0]['attributes']["email"] == klaviyo_identity_email
+    assert (
+        v[f"{dataset_name}:profiles"][0]["attributes"]["email"]
+        == klaviyo_identity_email
+    )
     user_id = v[f"{dataset_name}:profiles"][0]["id"]
 
 
@@ -107,13 +105,7 @@ async def test_klaviyo_erasure_request_task(
     assert_rows_match(
         v[f"{dataset_name}:profiles"],
         min_size=1,
-        keys=[
-            "type",
-            "id",
-            "attributes",
-            "links",
-            "relationships"
-        ],
+        keys=["type", "id", "attributes", "links", "relationships"],
     )
 
     x = await graph_task.run_erasure(
@@ -134,16 +126,16 @@ async def test_klaviyo_erasure_request_task(
     base_url = f"https://{klaviyo_secrets['domain']}"
     headers = {
         "Authorization": f"Klaviyo-API-Key {klaviyo_secrets['api_key']}",
-        "revision": klaviyo_secrets['revision']
+        "revision": klaviyo_secrets["revision"],
     }
 
     # user
     response = requests.get(
         url=f"{base_url}/api/profiles",
         headers=headers,
-        params={"filter": "equals(email,'"+klaviyo_erasure_identity_email+"')"},
+        params={"filter": "equals(email,'" + klaviyo_erasure_identity_email + "')"},
     )
-    
+
     assert response.status_code == 200
 
     CONFIG.execution.masking_strict = masking_strict

tests/ops/task/test_graph_task.py

@@ -1,8 +1,10 @@
 from typing import Any, Dict
+from unittest import mock
 
 import pytest
 from bson import ObjectId
 
+from fides.api.ops.common_exceptions import SkippingConsentPropagation
 from fides.api.ops.graph.config import (
     ROOT_COLLECTION_ADDRESS,
     TERMINATOR_ADDRESS,
@@ -15,6 +17,7 @@
 from fides.api.ops.graph.traversal import Traversal, TraversalNode
 from fides.api.ops.models.connectionconfig import ConnectionConfig, ConnectionType
 from fides.api.ops.models.policy import ActionType, Policy, Rule, RuleTarget
+from fides.api.ops.models.privacy_request import ExecutionLog, ExecutionLogStatus
 from fides.api.ops.task.graph_task import (
     EMPTY_REQUEST,
     GraphTask,
@@ -25,6 +28,9 @@
     start_function,
     update_erasure_mapping_from_cache,
 )
+from fides.api.ops.util.consent_util import (
+    cache_initial_status_and_identities_for_consent_reporting,
+)
 
 from ..graph.graph_test_util import (
     MockMongoTask,
@@ -33,6 +39,7 @@
     erasure_policy,
     field,
     generate_field_list,
+    generate_node,
 )
 from .traversal_data import (
     combined_mongo_postgresql_graph,
@@ -659,3 +666,128 @@ def test_update_erasure_mapping_from_cache_with_data(self, dsk, task_resource):
         }  # a cache with the results of the ds_1 collection erasure
         update_erasure_mapping_from_cache(dsk, task_resource)
         assert dsk[CollectionAddress("dr_1", "ds_1")] == 1
+
+
+class TestGraphTaskAffectedConsentSystems:
+    @pytest.fixture()
+    def mock_graph_task(
+        self,
+        db,
+        mailchimp_transactional_connection_config_no_secrets,
+        privacy_request_with_consent_policy,
+    ):
+        task_resources = TaskResources(
+            privacy_request_with_consent_policy,
+            privacy_request_with_consent_policy.policy,
+            [mailchimp_transactional_connection_config_no_secrets],
+            db,
+        )
+        tn = TraversalNode(generate_node("a", "b", "c", "c2"))
+        tn.node.dataset.connection_key = (
+            mailchimp_transactional_connection_config_no_secrets.key
+        )
+        return GraphTask(tn, task_resources)
+
+    @mock.patch(
+        "fides.api.ops.service.connectors.saas_connector.SaaSConnector.run_consent_request"
+    )
+    def test_skipped_consent_task_for_connector(
+        self,
+        mock_run_consent_request,
+        mock_graph_task,
+        db,
+        privacy_request_with_consent_policy,
+        privacy_preference_history,
+        privacy_preference_history_us_ca_provide,
+    ):
+        """Test that all privacy preferences for a consent connector get marked as skipped if SkippingConsentPropagation gets called"""
+        privacy_preference_history.privacy_request_id = (
+            privacy_request_with_consent_policy.id
+        )
+        privacy_preference_history_us_ca_provide.privacy_request_id = (
+            privacy_request_with_consent_policy.id
+        )
+        privacy_preference_history.save(db)
+        privacy_preference_history_us_ca_provide.save(db)
+
+        mock_run_consent_request.side_effect = SkippingConsentPropagation(
+            "No preferences are relevant"
+        )
+
+        ret = mock_graph_task.consent_request({"email": "[email protected]"})
+        assert ret is False
+
+        db.refresh(privacy_preference_history)
+        db.refresh(privacy_preference_history_us_ca_provide)
+
+        assert privacy_preference_history.affected_system_status == {
+            "mailchimp_transactional_instance": "skipped"
+        }
+        assert privacy_preference_history_us_ca_provide.affected_system_status == {
+            "mailchimp_transactional_instance": "skipped"
+        }
+
+        logs = (
+            db.query(ExecutionLog)
+            .filter(
+                ExecutionLog.privacy_request_id
+                == privacy_request_with_consent_policy.id
+            )
+            .order_by(ExecutionLog.created_at.desc())
+        )
+        assert logs.first().status == ExecutionLogStatus.skipped
+
+    @mock.patch(
+        "fides.api.ops.service.connectors.saas_connector.SaaSConnector.run_consent_request"
+    )
+    def test_errored_consent_task_for_connector_no_relevant_preferences(
+        self,
+        mock_run_consent_request,
+        mailchimp_transactional_connection_config_no_secrets,
+        mock_graph_task,
+        db,
+        privacy_request_with_consent_policy,
+        privacy_preference_history,
+        privacy_preference_history_us_ca_provide,
+    ):
+        """Test privacy preferences only marked as errored if they have pending logs"""
+        privacy_preference_history.privacy_request_id = (
+            privacy_request_with_consent_policy.id
+        )
+        privacy_preference_history_us_ca_provide.privacy_request_id = (
+            privacy_request_with_consent_policy.id
+        )
+        privacy_preference_history.save(db)
+        privacy_preference_history_us_ca_provide.save(db)
+
+        mock_run_consent_request.side_effect = BaseException("Request failed")
+        cache_initial_status_and_identities_for_consent_reporting(
+            db,
+            privacy_request_with_consent_policy,
+            mailchimp_transactional_connection_config_no_secrets,
+            relevant_preferences=[privacy_preference_history_us_ca_provide],
+            relevant_user_identities={"email": "[email protected]"},
+        )
+        with pytest.raises(BaseException):
+            ret = mock_graph_task.consent_request({"email": "[email protected]"})
+            assert ret is False
+
+        db.refresh(privacy_preference_history)
+        db.refresh(privacy_preference_history_us_ca_provide)
+
+        assert privacy_preference_history.affected_system_status == {
+            "mailchimp_transactional_instance": "skipped"
+        }
+        assert privacy_preference_history_us_ca_provide.affected_system_status == {
+            "mailchimp_transactional_instance": "error"
+        }
+
+        logs = (
+            db.query(ExecutionLog)
+            .filter(
+                ExecutionLog.privacy_request_id
+                == privacy_request_with_consent_policy.id
+            )
+            .order_by(ExecutionLog.created_at.desc())
+        )
+        assert logs.first().status == ExecutionLogStatus.error