diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index ffc19421d4..35f25f2c46 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -1018,6 +1018,10 @@ dataset: - name: updated_at data_categories: [system.operations] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_request_id + description: 'An optional link to the privacy request if one was created to propagate request preferences' + data_categories: [ system.operations ] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: datasetconfig data_categories: [] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified @@ -1279,6 +1283,9 @@ dataset: - name: updated_at data_categories: [system.operations] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: consent_preferences + data_categories: [ system.operations ] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: privacyrequesterror data_categories: [] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified diff --git a/clients/privacy-center/config/config.json b/clients/privacy-center/config/config.json index a2050cf00d..60448da8c3 100644 --- a/clients/privacy-center/config/config.json +++ b/clients/privacy-center/config/config.json @@ -37,6 +37,7 @@ "email": "required", "phone": "optional" }, + "policy_key": "default_consent_policy", "consentOptions": [ { "fidesDataUseKey": "advertising", @@ -45,7 +46,8 @@ "url": "https://example.com/privacy#data-sales", "default": true, "highlight": false, - "cookieKeys": ["data_sales"] + "cookieKeys": ["data_sales"], + "executable": true }, { "fidesDataUseKey": "advertising.first_party", @@ -54,7 +56,8 @@ "url": "https://example.com/privacy#email-marketing", "default": true, "highlight": false, - "cookieKeys": [] + "cookieKeys": [], + "executable": true }, { "fidesDataUseKey": "improve", @@ -63,7 +66,8 @@ "url": "https://example.com/privacy#analytics", "default": true, "highlight": false, - "cookieKeys": [] + "cookieKeys": [], + "executable": true } ] } diff --git a/src/fides/api/ctl/database/seed.py b/src/fides/api/ctl/database/seed.py index 2f6f7e25b3..274d67a12a 100644 --- a/src/fides/api/ctl/database/seed.py +++ b/src/fides/api/ctl/database/seed.py @@ -41,6 +41,9 @@ DEFAULT_ERASURE_POLICY_RULE = "default_erasure_policy_rule" DEFAULT_ERASURE_MASKING_STRATEGY = "hmac" +DEFAULT_CONSENT_POLICY: str = "default_consent_policy" +DEFAULT_CONSENT_RULE = "default_consent_rule" + def create_or_update_parent_user() -> None: with sync_session() as db_session: @@ -278,6 +281,30 @@ async def load_default_dsr_policies() -> None: except KeyOrNameAlreadyExists: # This rule target already exists against the Policy pass + + log.info("Creating: Default Consent Policy") + consent_policy = Policy.create_or_update( + db=db_session, + data={ + "name": "Default Consent Policy", + "key": DEFAULT_CONSENT_POLICY, + "execution_timeframe": 45, + "client_id": client_id, + }, + ) + + log.info("Creating: Default Consent Rule") + Rule.create_or_update( + db=db_session, + data={ + "action_type": ActionType.consent.value, + "name": "Default Consent Rule", + "key": DEFAULT_CONSENT_RULE, + "policy_id": consent_policy.id, + "client_id": client_id, + }, + ) + log.info("All Policies & Rules Seeded.") diff --git a/src/fides/api/ctl/migrations/versions/de456534dbda_add_privacyrequest_consent_preferences_.py b/src/fides/api/ctl/migrations/versions/de456534dbda_add_privacyrequest_consent_preferences_.py new file mode 100644 index 0000000000..6755183a24 --- /dev/null +++ b/src/fides/api/ctl/migrations/versions/de456534dbda_add_privacyrequest_consent_preferences_.py @@ -0,0 +1,43 @@ +"""add privacyrequest consent preferences and ruleuse table + +Revision ID: de456534dbda +Revises: 3caf11127442 +Create Date: 2023-01-03 22:59:45.144538 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "de456534dbda" +down_revision = "3caf11127442" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "privacyrequest", + sa.Column( + "consent_preferences", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + ), + ) + + op.add_column( + "consentrequest", sa.Column("privacy_request_id", sa.String(), nullable=True) + ) + op.create_foreign_key( + None, "consentrequest", "privacyrequest", ["privacy_request_id"], ["id"] + ) + + +def downgrade(): + op.drop_column("privacyrequest", "consent_preferences") + + op.drop_constraint( + "consentrequest_privacy_request_id_fkey", "consentrequest", type_="foreignkey" + ) + op.drop_column("consentrequest", "privacy_request_id") diff --git a/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py b/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py index 2aea41c84b..1500f36937 100644 --- a/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import Optional +import json +from typing import Dict, List, Optional, Tuple, Union from fastapi import Depends, HTTPException, Security from loguru import logger @@ -15,7 +16,11 @@ HTTP_500_INTERNAL_SERVER_ERROR, ) +from fides.api.ctl.database.seed import DEFAULT_CONSENT_POLICY from fides.api.ops.api.deps import get_db +from fides.api.ops.api.v1.endpoints.privacy_request_endpoints import ( + create_privacy_request_func, +) from fides.api.ops.api.v1.scope_registry import CONSENT_READ from fides.api.ops.api.v1.urn_registry import ( CONSENT_REQUEST, @@ -37,14 +42,17 @@ ProvidedIdentityType, ) from fides.api.ops.schemas.messaging.messaging import MessagingMethod +from fides.api.ops.schemas.privacy_request import BulkPostPrivacyRequests from fides.api.ops.schemas.privacy_request import Consent as ConsentSchema from fides.api.ops.schemas.privacy_request import ( ConsentPreferences, ConsentPreferencesWithVerificationCode, ConsentRequestResponse, + PrivacyRequestCreate, VerificationCode, ) from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.schemas.shared_schemas import FidesOpsKey from fides.api.ops.service._verification import send_verification_code_to_user from fides.api.ops.util.api_router import APIRouter from fides.api.ops.util.logger import Pii @@ -54,6 +62,7 @@ router = APIRouter(tags=["Consent"], prefix=V1_URL_PREFIX) CONFIG = get_config() +CONFIG_JSON_PATH = "clients/privacy-center/config/config.json" @router.post( @@ -115,7 +124,7 @@ def consent_request_verify( data: VerificationCode, ) -> ConsentPreferences: """Verifies the verification code and returns the current consent preferences if successful.""" - provided_identity = _get_consent_request_and_provided_identity( + _, provided_identity = _get_consent_request_and_provided_identity( db=db, consent_request_id=consent_request_id, verification_code=data.code ) @@ -171,7 +180,7 @@ def get_consent_preferences_no_id( "turned off.", ) - provided_identity = _get_consent_request_and_provided_identity( + _, provided_identity = _get_consent_request_and_provided_identity( db=db, consent_request_id=consent_request_id, verification_code=None ) @@ -216,6 +225,86 @@ def get_consent_preferences( return _prepare_consent_preferences(db, identity) +def load_executable_consent_options(file_path: str) -> List[str]: + """Load customer's consentOptions from the config.json file and filter to return only a list + of executable consent options""" + with open(file_path, encoding="utf-8") as privacy_center_config_file: + privacy_center_config: Dict = json.load(privacy_center_config_file) + consent_options: List = privacy_center_config.get("consent", {}).get( + "consentOptions", [] + ) + + executable_consent_options: List[str] = [] + for consent in consent_options: + data_use: str = consent.get("fidesDataUseKey") + if not data_use: + continue + + if consent.get("executable"): + executable_consent_options.append(data_use) + else: + logger.info("Consent option: '{}' is not executable.", data_use) + + return executable_consent_options + + +def queue_privacy_request_to_propagate_consent( + db: Session, + provided_identity: ProvidedIdentity, + policy: Union[FidesOpsKey, str], + consent_preferences: ConsentPreferences, +) -> Optional[BulkPostPrivacyRequests]: + """ + Queue a privacy request to carry out propagating consent preferences server-side to third-party systems. + + Only propagate consent preferences which are considered "executable" by the current system. If none of the + consent preferences are executable, no Privacy Request is queued. + """ + identity = Identity() + setattr( + identity, + provided_identity.field_name.value, # type:ignore[attr-defined] + provided_identity.encrypted_value["value"], # type:ignore[index] + ) # Pull the information on the ProvidedIdentity for the ConsentRequest to pass along to create a PrivacyRequest + + executable_consent_options: List[str] = load_executable_consent_options( + CONFIG_JSON_PATH + ) + executable_consent_preferences: List[Dict] = [ + pref.dict() + for pref in consent_preferences.consent or [] + if pref.data_use in executable_consent_options + ] + + if not executable_consent_preferences: + logger.info( + "Skipping propagating consent preferences to third-party services as " + "specified consent preferences: {} are not executable.", + [pref.data_use for pref in consent_preferences.consent or []], + ) + return None + + privacy_request_results: BulkPostPrivacyRequests = create_privacy_request_func( + db=db, + data=[ + PrivacyRequestCreate( + identity=identity, + policy_key=policy, + consent_preferences=executable_consent_preferences, + ) + ], + authenticated=True, + ) + + if privacy_request_results.failed or not privacy_request_results.succeeded: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=privacy_request_results.failed[0].message, + ) + + return privacy_request_results + + @router.patch( CONSENT_REQUEST_PREFERENCES_WITH_ID, status_code=HTTP_200_OK, @@ -228,7 +317,7 @@ def set_consent_preferences( data: ConsentPreferencesWithVerificationCode, ) -> ConsentPreferences: """Verifies the verification code and saves the user's consent preferences if successful.""" - provided_identity = _get_consent_request_and_provided_identity( + consent_request, provided_identity = _get_consent_request_and_provided_identity( db=db, consent_request_id=consent_request_id, verification_code=data.code, @@ -258,7 +347,27 @@ def set_consent_preferences( status_code=HTTP_400_BAD_REQUEST, detail=Pii(str(exc)) ) - return _prepare_consent_preferences(db, provided_identity) + consent_preferences: ConsentPreferences = _prepare_consent_preferences( + db, provided_identity + ) + + # Note: This just queues the PrivacyRequest for processing + privacy_request_creation_results: Optional[ + BulkPostPrivacyRequests + ] = queue_privacy_request_to_propagate_consent( + db, + provided_identity, + data.policy_key or DEFAULT_CONSENT_POLICY, + consent_preferences, + ) + + if privacy_request_creation_results: + consent_request.privacy_request_id = privacy_request_creation_results.succeeded[ + 0 + ].id + consent_request.save(db=db) + + return consent_preferences def _get_or_create_provided_identity( @@ -355,7 +464,7 @@ def _get_consent_request_and_provided_identity( db: Session, consent_request_id: str, verification_code: Optional[str], -) -> ProvidedIdentity: +) -> Tuple[ConsentRequest, ProvidedIdentity]: """Verifies the consent request and verification code, then return the ProvidedIdentity if successful.""" consent_request = ConsentRequest.get_by_key_or_id( db=db, data={"id": consent_request_id} @@ -389,7 +498,7 @@ def _get_consent_request_and_provided_identity( detail="No identity found for consent request id", ) - return provided_identity + return consent_request, provided_identity def _prepare_consent_preferences( diff --git a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py index c68499ad0c..d2600f40e4 100644 --- a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py @@ -187,7 +187,7 @@ def create_privacy_request( or report failure and execute them within the Fidesops system. You cannot update privacy requests after they've been created. """ - return _create_privacy_request(db, data, False) + return create_privacy_request_func(db, data, False) @router.post( @@ -207,7 +207,7 @@ def create_privacy_request_authenticated( You cannot update privacy requests after they've been created. This route requires authentication instead of using verification codes. """ - return _create_privacy_request(db, data, True) + return create_privacy_request_func(db, data, True) def _send_privacy_request_receipt_message_to_user( @@ -1553,7 +1553,7 @@ def resume_privacy_request_from_requires_input( return privacy_request -def _create_privacy_request( +def create_privacy_request_func( db: Session, data: conlist(PrivacyRequestCreate), # type: ignore authenticated: bool = False, @@ -1574,7 +1574,12 @@ def _create_privacy_request( logger.info("Starting creation for {} privacy requests", len(data)) - optional_fields = ["external_id", "started_processing_at", "finished_processing_at"] + optional_fields = [ + "external_id", + "started_processing_at", + "finished_processing_at", + "consent_preferences", + ] for privacy_request_data in data: if not any(privacy_request_data.identity.dict().values()): logger.warning( @@ -1612,6 +1617,9 @@ def _create_privacy_request( for field in optional_fields: attr = getattr(privacy_request_data, field) if attr is not None: + if field == "consent_preferences": + attr = [consent.dict() for consent in attr] + kwargs[field] = attr try: diff --git a/src/fides/api/ops/common_exceptions.py b/src/fides/api/ops/common_exceptions.py index 748666d1c8..76bd2d8998 100644 --- a/src/fides/api/ops/common_exceptions.py +++ b/src/fides/api/ops/common_exceptions.py @@ -101,6 +101,10 @@ class CollectionDisabled(BaseException): """Collection is attached to disabled ConnectionConfig""" +class NotSupportedForCollection(BaseException): + """The given action is not supported for this type of collection""" + + class PrivacyRequestPaused(BaseException): """Halt Instruction Received on Privacy Request""" diff --git a/src/fides/api/ops/models/datasetconfig.py b/src/fides/api/ops/models/datasetconfig.py index 59d8b2686c..87411a1371 100644 --- a/src/fides/api/ops/models/datasetconfig.py +++ b/src/fides/api/ops/models/datasetconfig.py @@ -96,6 +96,24 @@ def get_graph(self) -> Dataset: "Connection config with key {} is not a saas config, skipping merge dataset", self.connection_config.key, ) + + return dataset_graph + + def get_dataset_with_stubbed_collection(self) -> Dataset: + """ + Return a Dataset with a single mock Collection for use in building a graph + where we only want one node per dataset, instead of one node per collection. Note that + the expectation is that there would be no dependencies between nodes on the eventual graph, and the graph + doesn't require information stored at the collection-level. + + The single Collection will be the resource that gets practically added to the graph, but the intent + is that this single node represents the overall Dataset, and will execute Dataset-level requests, + not Collection-level requests. + """ + dataset_graph: Dataset = self.get_graph() + stubbed_collection = Collection(name=dataset_graph.name, fields=[], after=set()) + + dataset_graph.collections = [stubbed_collection] return dataset_graph diff --git a/src/fides/api/ops/models/policy.py b/src/fides/api/ops/models/policy.py index 78277eeac1..c4a06e66ee 100644 --- a/src/fides/api/ops/models/policy.py +++ b/src/fides/api/ops/models/policy.py @@ -32,6 +32,7 @@ class CurrentStep(EnumType): pre_webhooks = "pre_webhooks" access = "access" erasure = "erasure" + consent = "consent" erasure_email_post_send = "erasure_email_post_send" post_webhooks = "post_webhooks" @@ -83,6 +84,7 @@ def _validate_rule( """Check that the rule's action_type and storage_destination are valid.""" if not action_type: raise common_exceptions.RuleValidationError("action_type is required.") + if action_type == ActionType.erasure.value: if storage_destination_id is not None: raise common_exceptions.RuleValidationError( @@ -97,7 +99,7 @@ def _validate_rule( raise common_exceptions.RuleValidationError( "Access Rules must have a storage destination." ) - if action_type in [ActionType.consent.value, ActionType.update.value]: + if action_type in [ActionType.update.value]: raise common_exceptions.RuleValidationError( f"{action_type} Rules are not supported at this time." ) @@ -151,6 +153,11 @@ def get_rules_for_action(self, action_type: ActionType) -> List["Rule"]: """Returns all Rules related to this Policy filtered by `action_type`.""" return [rule for rule in self.rules if rule.action_type == action_type] # type: ignore[attr-defined] + def get_consent_rule(self) -> Optional["Rule"]: + """Returns a Consent Rule if it exists. There should only be one.""" + consent_rules = self.get_rules_for_action(ActionType.consent) + return consent_rules[0] if consent_rules else None + def _get_ref_from_taxonomy(fides_key: FidesOpsKey) -> FideslangDataCategory: """Returns the DataCategory model from the DEFAULT_TAXONOMY corresponding to fides_key.""" @@ -269,7 +276,28 @@ def save(self, db: Session) -> FidesBase: @classmethod def create(cls, db: Session, *, data: Dict[str, Any]) -> FidesBase: # type: ignore[override] - """Validate this object's data before deferring to the superclass on update""" + """Validate this object's data before deferring to the superclass on create""" + policy_id: Optional[str] = data.get("policy_id") + + if not policy_id: + raise common_exceptions.RuleValidationError( + "Policy id must be specified on Rule create." + ) + + policy = Policy.get_by(db=db, field="id", value=policy_id) + if not policy: + raise common_exceptions.RuleValidationError( + "Policy id must be specified on Rule create." + ) + existing_consent_rules = policy.get_rules_for_action(ActionType.consent) + + if ( + existing_consent_rules + and data.get("action_type") == ActionType.consent.value + ): + raise common_exceptions.RuleValidationError( + f"Policies can only have one consent rule attached. Existing rule {existing_consent_rules[0].key} found." + ) _validate_rule( action_type=data.get("action_type"), storage_destination_id=data.get("storage_destination_id"), diff --git a/src/fides/api/ops/models/privacy_request.py b/src/fides/api/ops/models/privacy_request.py index e2641b1121..bbefa018ec 100644 --- a/src/fides/api/ops/models/privacy_request.py +++ b/src/fides/api/ops/models/privacy_request.py @@ -78,6 +78,7 @@ CurrentStep.pre_webhooks, CurrentStep.access, CurrentStep.erasure, + CurrentStep.consent, CurrentStep.erasure_email_post_send, CurrentStep.post_webhooks, ] @@ -190,6 +191,7 @@ class PrivacyRequest(IdentityVerificationMixin, Base): # pylint: disable=R0904 cancel_reason = Column(String(200)) canceled_at = Column(DateTime(timezone=True), nullable=True) + consent_preferences = Column(MutableList.as_mutable(JSONB), nullable=True) # passive_deletes="all" prevents execution logs from having their privacy_request_id set to null when # a privacy_request is deleted. We want to retain for record-keeping. @@ -855,6 +857,9 @@ class ConsentRequest(IdentityVerificationMixin, Base): back_populates="consent_request", ) + privacy_request_id = Column(String, ForeignKey(PrivacyRequest.id), nullable=True) + privacy_request = relationship(PrivacyRequest) + def get_cached_identity_data(self) -> Dict[str, Any]: """Retrieves any identity data pertaining to this request from the cache.""" prefix = f"id-{self.id}-identity-*" diff --git a/src/fides/api/ops/schemas/privacy_request.py b/src/fides/api/ops/schemas/privacy_request.py index 701724d8ac..81008c29a6 100644 --- a/src/fides/api/ops/schemas/privacy_request.py +++ b/src/fides/api/ops/schemas/privacy_request.py @@ -54,6 +54,14 @@ class Config: use_enum_values = True +class Consent(BaseSchema): + """Schema for consent.""" + + data_use: str + data_use_description: Optional[str] = None + opt_in: bool + + class PrivacyRequestCreate(BaseSchema): """Data required to create a PrivacyRequest""" @@ -64,6 +72,7 @@ class PrivacyRequestCreate(BaseSchema): identity: Identity policy_key: FidesOpsKey encryption_key: Optional[str] = None + consent_preferences: Optional[List[Consent]] = None @validator("encryption_key") def validate_encryption_key( @@ -227,16 +236,8 @@ class BulkReviewResponse(BulkPostPrivacyRequests): """Schema with mixed success/failure responses for Bulk Approve/Deny of PrivacyRequest responses.""" -class Consent(BaseSchema): - """Schema for consent.""" - - data_use: str - data_use_description: Optional[str] = None - opt_in: bool - - class ConsentPreferences(BaseSchema): - """Schema for consent prefernces.""" + """Schema for consent preferences.""" consent: Optional[List[Consent]] = None @@ -246,6 +247,7 @@ class ConsentPreferencesWithVerificationCode(BaseSchema): code: Optional[str] consent: List[Consent] + policy_key: Optional[FidesOpsKey] = None class ConsentRequestResponse(BaseSchema): diff --git a/src/fides/api/ops/service/connectors/base_connector.py b/src/fides/api/ops/service/connectors/base_connector.py index f9379e6c17..47cd4bfe1b 100644 --- a/src/fides/api/ops/service/connectors/base_connector.py +++ b/src/fides/api/ops/service/connectors/base_connector.py @@ -1,10 +1,11 @@ from abc import ABC, abstractmethod from typing import Any, Dict, Generic, List, Optional, TypeVar +from fides.api.ops.common_exceptions import NotSupportedForCollection from fides.api.ops.graph.traversal import TraversalNode from fides.api.ops.models.connectionconfig import ConnectionConfig, ConnectionTestStatus from fides.api.ops.models.policy import Policy -from fides.api.ops.models.privacy_request import PrivacyRequest +from fides.api.ops.models.privacy_request import Consent, PrivacyRequest from fides.api.ops.service.connectors.query_config import QueryConfig from fides.api.ops.util.collection_util import Row from fides.core.config import get_config @@ -87,6 +88,23 @@ def mask_data( was passed into "retrieve_data" for use in querying for data. """ + def run_consent_request( + self, + node: TraversalNode, + policy: Policy, + privacy_request: PrivacyRequest, + identity_data: Dict[str, Any], + executable_preferences: List[Consent], + ) -> bool: + """ + Base method for executing a consent request. Override on a given connector if functionality + is supported. Otherwise, this collections with this connector type will be skipped. + + """ + raise NotSupportedForCollection( + f"Consent requests are not supported for connectors of type {self.configuration.connection_type}" + ) + def dry_run_query(self, node: TraversalNode) -> Optional[str]: """Generate a dry-run query to display action that will be taken""" return self.query_config(node).dry_run_query() diff --git a/src/fides/api/ops/service/connectors/saas_connector.py b/src/fides/api/ops/service/connectors/saas_connector.py index 671e115917..6a47558433 100644 --- a/src/fides/api/ops/service/connectors/saas_connector.py +++ b/src/fides/api/ops/service/connectors/saas_connector.py @@ -9,7 +9,7 @@ from fides.api.ops.graph.traversal import TraversalNode from fides.api.ops.models.connectionconfig import ConnectionConfig, ConnectionTestStatus from fides.api.ops.models.policy import Policy -from fides.api.ops.models.privacy_request import PrivacyRequest +from fides.api.ops.models.privacy_request import Consent, PrivacyRequest from fides.api.ops.schemas.limiter.rate_limit_config import RateLimitConfig from fides.api.ops.schemas.saas.saas_config import ClientConfig, ParamValue, SaaSRequest from fides.api.ops.schemas.saas.shared_schemas import SaaSRequestParams @@ -404,6 +404,35 @@ def mask_data( self.unset_connector_state() return rows_updated + def run_consent_request( + self, + node: TraversalNode, + policy: Policy, + privacy_request: PrivacyRequest, + identity_data: Dict[str, Any], + executable_preferences: List[Consent], + ) -> bool: + """Execute a consent request. Return whether the consent request to the third party succeeded. + + Executable_preferences have already been filtered to just consent preferences the customer has deemed executable. + + Return True if 200 OK + """ + logger.info( + "Mocking consent request - actual logic should go here for saas connectors", + node.address.value, + ) + + logger.info( + "Demo only! Testing available consent params! identity_data: {}, executable_preferences: {}", + identity_data, + [ + {"use": preference.data_use, "opt_in": preference.opt_in} + for preference in executable_preferences + ], + ) + return True + def close(self) -> None: """Not required for this type""" diff --git a/src/fides/api/ops/service/privacy_request/request_runner_service.py b/src/fides/api/ops/service/privacy_request/request_runner_service.py index cff6ec1e73..380e64cb4a 100644 --- a/src/fides/api/ops/service/privacy_request/request_runner_service.py +++ b/src/fides/api/ops/service/privacy_request/request_runner_service.py @@ -25,9 +25,9 @@ failed_graph_analytics_event, fideslog_graph_failure, ) -from fides.api.ops.graph.config import CollectionAddress +from fides.api.ops.graph.config import CollectionAddress, Dataset from fides.api.ops.graph.graph import DatasetGraph -from fides.api.ops.models.connectionconfig import ConnectionConfig +from fides.api.ops.models.connectionconfig import ConnectionConfig, ConnectionType from fides.api.ops.models.datasetconfig import DatasetConfig from fides.api.ops.models.manual_webhook import AccessManualWebhook from fides.api.ops.models.policy import ( @@ -62,6 +62,7 @@ from fides.api.ops.task.graph_task import ( get_cached_data_for_erasures, run_access_request, + run_consent_request, run_erasure, ) from fides.api.ops.tasks import DatabaseTask, celery_app @@ -101,7 +102,7 @@ def get_access_manual_webhook_inputs( manual_inputs: Dict[str, List[Dict[str, Optional[Any]]]] = {} if not policy.get_rules_for_action(action_type=ActionType.access): - # Don't fetch manual inputs if this is an erasure-only request + # Don't fetch manual inputs unless this policy has an access rule return ManualWebhookResults(manual_data=manual_inputs, proceed=True) try: @@ -340,7 +341,10 @@ async def run_privacy_request( ) access_result_urls: List[str] = [] - if can_run_checkpoint( + if ( + policy.get_rules_for_action(action_type=ActionType.access) + or policy.get_rules_for_action(action_type=ActionType.erasure) + ) and can_run_checkpoint( request_checkpoint=CurrentStep.access, from_checkpoint=resume_step ): access_result: Dict[str, List[Row]] = await run_access_request( @@ -379,6 +383,21 @@ async def run_privacy_request( session=session, ) + if policy.get_rules_for_action( + action_type=ActionType.consent + ) and can_run_checkpoint( + request_checkpoint=CurrentStep.consent, + from_checkpoint=resume_step, + ): + await run_consent_request( + privacy_request=privacy_request, + policy=policy, + graph=build_consent_dataset_graph(datasets), + connection_configs=connection_configs, + identity=identity_data, + session=session, + ) + except PrivacyRequestPaused as exc: privacy_request.pause_processing(session) _log_warning(exc, CONFIG.dev_mode) @@ -395,7 +414,9 @@ async def run_privacy_request( return # Send erasure requests via email to third parties where applicable - if can_run_checkpoint( + if policy.get_rules_for_action( + action_type=ActionType.erasure + ) and can_run_checkpoint( request_checkpoint=CurrentStep.erasure_email_post_send, from_checkpoint=resume_step, ): @@ -423,6 +444,7 @@ async def run_privacy_request( ) if not proceed: return + if CONFIG.notifications.send_request_completion_notification: try: initiate_privacy_request_completion_email( @@ -451,6 +473,22 @@ async def run_privacy_request( privacy_request.save(db=session) +def build_consent_dataset_graph(datasets: List[DatasetConfig]) -> DatasetGraph: + """ + Build the starting DatasetGraph for consent requests. + + Returns a DatasetGraph built from Datasets that have a single mocked collection, for when we want one node + per dataset, with no nodes per dependencies. For now, the resource must be a "saas" type but this + is subject to change. + """ + datasets_with_representative_collections: List[Dataset] = [ + dataset_config.get_dataset_with_stubbed_collection() + for dataset_config in datasets + if dataset_config.connection_config.connection_type == ConnectionType.saas + ] + return DatasetGraph(*datasets_with_representative_collections) + + def initiate_privacy_request_completion_email( session: Session, policy: Policy, diff --git a/src/fides/api/ops/task/graph_task.py b/src/fides/api/ops/task/graph_task.py index 1b45949a6a..74790d0d69 100644 --- a/src/fides/api/ops/task/graph_task.py +++ b/src/fides/api/ops/task/graph_task.py @@ -13,6 +13,7 @@ from fides.api.ops.common_exceptions import ( CollectionDisabled, + NotSupportedForCollection, PrivacyRequestErasureEmailSendRequired, PrivacyRequestPaused, ) @@ -32,8 +33,12 @@ from fides.api.ops.graph.graph_differences import format_graph_for_caching from fides.api.ops.graph.traversal import Traversal, TraversalNode from fides.api.ops.models.connectionconfig import AccessLevel, ConnectionConfig -from fides.api.ops.models.policy import ActionType, Policy -from fides.api.ops.models.privacy_request import ExecutionLogStatus, PrivacyRequest +from fides.api.ops.models.policy import ActionType, Policy, Rule +from fides.api.ops.models.privacy_request import ( + Consent, + ExecutionLogStatus, + PrivacyRequest, +) from fides.api.ops.service.connectors.base_connector import BaseConnector from fides.api.ops.task.consolidate_query_matches import consolidate_query_matches from fides.api.ops.task.filter_element_match import filter_element_match @@ -98,9 +103,9 @@ def result(*args: Any, **kwargs: Any) -> Any: f"{self.traversal_node.address.value}", 0 ) # Cache that the erasure was performed in case we need to restart return 0 - except CollectionDisabled as exc: + except (CollectionDisabled, NotSupportedForCollection) as exc: logger.warning( - "Skipping disabled collection {} for privacy_request: {}", + "Skipping collection {} for privacy_request: {}", self.traversal_node.address, self.resources.request.id, ) @@ -554,6 +559,38 @@ def erasure_request(self, retrieved_data: List[Row], *inputs: List[Row]) -> int: ) # Cache that the erasure was performed in case we need to restart return output + @retry(action_type=ActionType.consent, default_return=False) + def consent_request(self, identity: Dict[str, Any]) -> bool: + """Run consent request request""" + + if not self.can_write_data(): + logger.warning( + "No consent on {} as its ConnectionConfig does not have write access.", + self.traversal_node.node.address, + ) + self.update_status( + f"No values were erased since this connection {self.connector.configuration.key} has not been " + f"given write access", + None, + ActionType.erasure, + ExecutionLogStatus.error, + ) + return False + + consent_preferences: List[Consent] = [ + Consent(**pref) for pref in self.resources.request.consent_preferences or [] + ] + + output: bool = self.connector.run_consent_request( + self.traversal_node, + self.resources.policy, + self.resources.request, + identity, + consent_preferences, + ) + self.log_end(ActionType.consent) + return output + def collect_queries( traversal: Traversal, resources: TaskResources @@ -758,6 +795,57 @@ def termination_fn(*dependent_values: int) -> Tuple[int, ...]: return erasure_update_map +async def run_consent_request( # pylint: disable = too-many-arguments + privacy_request: PrivacyRequest, + policy: Policy, + graph: DatasetGraph, + connection_configs: List[ConnectionConfig], + identity: Dict[str, Any], + session: Session, +) -> Dict[str, bool]: + """Run a consent request + + The graph built is very simple: there are no relationships between the nodes, every node has + identity data input and every node outputs whether the consent request succeeded. + + The DatasetGraph passed in is expected to have one Node per Dataset. That Node is expected to carry out requests + for the Dataset as a whole. + """ + + with TaskResources( + privacy_request, policy, connection_configs, session + ) as resources: + graph_keys: List[CollectionAddress] = list(graph.nodes.keys()) + dsk: Dict[CollectionAddress, Any] = {} + + for col_address, node in graph.nodes.items(): + traversal_node = TraversalNode(node) + task = GraphTask(traversal_node, resources) + dsk[col_address] = (task.consent_request, identity) + + def termination_fn(*dependent_values: bool) -> Tuple[bool, ...]: + """The dependent_values here is an bool output from each task feeding in, where + each task reports the output of 'task.consent_request(identity_data)', which is whether the + consent request succeeded + + The termination function just returns this tuple of booleans.""" + return dependent_values + + # terminator function waits for all keys + dsk[TERMINATOR_ADDRESS] = (termination_fn, *graph_keys) + + v = delayed(get(dsk, TERMINATOR_ADDRESS, num_workers=1)) + + update_successes: Tuple[bool, ...] = v.compute() + # we combine the output of the termination function with the input keys to provide + # a map of {collection_name: whether consent request succeeded}: + consent_update_map: Dict[str, bool] = dict( + zip([coll.value for coll in graph_keys], update_successes) + ) + + return consent_update_map + + def build_affected_field_logs( node: Node, policy: Policy, action_type: ActionType ) -> List[Dict[str, Any]]: diff --git a/src/fides/data/test_env/privacy_center_config/config.json b/src/fides/data/test_env/privacy_center_config/config.json index eab0b87dd1..2438df08fd 100644 --- a/src/fides/data/test_env/privacy_center_config/config.json +++ b/src/fides/data/test_env/privacy_center_config/config.json @@ -34,6 +34,7 @@ "title": "Manage your consent", "description": "Manage your consent preferences, including the option to select 'Do Not Sell My Personal Information'.", "cookieName": "fides_consent", + "policy_key": "default_consent_policy", "consentOptions": [ { "fidesDataUseKey": "advertising", diff --git a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py index e7a9046afe..aafb7b196f 100644 --- a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py @@ -2,9 +2,11 @@ from copy import deepcopy from typing import Any +from unittest import mock from unittest.mock import MagicMock, patch import pytest +from requests import Session from fides.api.ops.api.v1.scope_registry import CONNECTION_READ, CONSENT_READ from fides.api.ops.api.v1.urn_registry import ( @@ -519,11 +521,16 @@ def test_set_consent_preferences_invalid_code( @pytest.mark.usefixtures( "subject_identity_verification_required", ) + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) def test_verify_then_set_consent_preferences( self, + run_privacy_request_mock, provided_identity_and_consent_request, api_client, verification_code, + db: Session, ): _, consent_request = provided_identity_and_consent_request consent_request.cache_identity_verification_code(verification_code) @@ -545,9 +552,14 @@ def test_verify_then_set_consent_preferences( }, ) assert response.status_code == 200 - # Assert nconsent preferences have successfully been set assert response.json()["consent"][0]["data_use"] == "email" - assert response.json()["consent"][0]["opt_in"] == True + assert response.json()["consent"][0]["opt_in"] is True + assert not run_privacy_request_mock.called, "date_use: email is not executable" + + db.refresh(consent_request) + assert ( + not consent_request.privacy_request_id + ), "No PrivacyRequest queued because none of the consent options are executable" response = api_client.post( f"{V1_URL_PREFIX}{CONSENT_REQUEST_VERIFY.format(consent_request_id=consent_request.id)}", @@ -556,7 +568,7 @@ def test_verify_then_set_consent_preferences( assert response.status_code == 200 # Assert the code verification endpoint also returns existing consent preferences assert response.json()["consent"][0]["data_use"] == "email" - assert response.json()["consent"][0]["opt_in"] == True + assert response.json()["consent"][0]["opt_in"] is True @pytest.mark.usefixtures( "subject_identity_verification_required", @@ -659,27 +671,32 @@ def test_set_consent_preferences_no_consent_present( "subject_identity_verification_required", ) @patch("fides.api.ops.models.privacy_request.ConsentRequest.verify_identity") + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) def test_set_consent_consent_preferences( self, + mock_run_privacy_request: MagicMock, mock_verify_identity: MagicMock, provided_identity_and_consent_request, db, api_client, verification_code, + consent_policy, ): provided_identity, consent_request = provided_identity_and_consent_request consent_request.cache_identity_verification_code(verification_code) consent_data: list[dict[str, Any]] = [ { - "data_use": "email", + "data_use": "advertising", # Must match consentOptions in config.json "data_use_description": None, "opt_in": True, }, { - "data_use": "location", - "data_use_description": "Location data", - "opt_in": False, + "data_use": "improve", + "data_use_description": None, + "opt_in": True, }, ] @@ -693,6 +710,7 @@ def test_set_consent_consent_preferences( "code": verification_code, "identity": {"email": "test@email.com"}, "consent": consent_data, + "policy_key": consent_policy.key, # Optional policy_key supplied } response = api_client.patch( f"{V1_URL_PREFIX}{CONSENT_REQUEST_PREFERENCES_WITH_ID.format(consent_request_id=consent_request.id)}", @@ -703,6 +721,24 @@ def test_set_consent_consent_preferences( assert response.json()["consent"] == consent_data mock_verify_identity.assert_called_with(verification_code) + db.refresh(consent_request) + assert ( + consent_request.privacy_request_id is not None + ), "PrivacyRequest queued to propagate consent preferences cached on ConsentRequest" + + identity = consent_request.privacy_request.get_persisted_identity() + assert identity.email == "test@email.com", ( + "Identity pulled from Consent Provided Identity and used to " + "create a Privacy Request provided identity " + ) + assert identity.phone_number is None + assert consent_request.privacy_request.consent_preferences == [ + {"opt_in": True, "data_use": "advertising", "data_use_description": None}, + {"opt_in": False, "data_use": "improve", "data_use_description": None}, + ], "Only executable consent preferences stored" + + assert mock_run_privacy_request.called + @patch("fides.api.ops.models.privacy_request.ConsentRequest.verify_identity") def test_set_consent_consent_preferences_without_verification( self, diff --git a/tests/ops/fixtures/application_fixtures.py b/tests/ops/fixtures/application_fixtures.py index f74b15d6e7..06473ea009 100644 --- a/tests/ops/fixtures/application_fixtures.py +++ b/tests/ops/fixtures/application_fixtures.py @@ -37,6 +37,7 @@ MessagingServiceType, ) from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.schemas.saas.saas_config import ClientConfig, SaaSConfig, SaaSRequest from fides.api.ops.schemas.storage.storage import ( FileNaming, S3AuthMethod, @@ -667,6 +668,43 @@ def policy( pass +@pytest.fixture(scope="function") +def consent_policy( + db: Session, + oauth_client: ClientDetail, + storage_config: StorageConfig, +) -> Generator: + """Consent policies only need a ConsentRule attached - no RuleTargets necessary""" + consent_request_policy = Policy.create( + db=db, + data={ + "name": "example consent request policy", + "key": "example_consent_request_policy", + "client_id": oauth_client.id, + }, + ) + + consent_request_rule = Rule.create( + db=db, + data={ + "action_type": ActionType.consent.value, + "client_id": oauth_client.id, + "name": "Consent Request Rule", + "policy_id": consent_request_policy.id, + }, + ) + + yield consent_request_policy + try: + consent_request_rule.delete(db) + except ObjectDeletedError: + pass + try: + consent_request_policy.delete(db) + except ObjectDeletedError: + pass + + @pytest.fixture(scope="function") def policy_local_storage( db: Session, @@ -1481,3 +1519,59 @@ def authenticated_fides_client( ) -> FidesClient: test_fides_client.login() return test_fides_client + + +@pytest.fixture(scope="function") +def base_saas_connection_config(db): + """Non-specific saas connection config for illustrating making consent requests + + This is not capable of making requests + """ + connection_config = ConnectionConfig.create( + db=db, + data={ + "name": str(uuid4()), + "key": "my_base_saas_config", + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "disabled": False, + "description": "Test saas connection", + "saas_config": SaaSConfig( + fides_key="my_base_saas_config", + name="test", + type="mandrill", + description="Test saas config", + version="1.0", + connector_params=[], + client_config=ClientConfig(protocol="test", host="example.com"), + endpoints=[], + test_request=SaaSRequest(path="example.com", method="POST"), + ).dict(), + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture(scope="function") +def base_saas_dataset_config( + base_saas_connection_config: ConnectionConfig, + db: Session, +) -> Generator: + """Non-specific saas DatasetConfig""" + dataset_config = DatasetConfig.create( + db=db, + data={ + "connection_config_id": base_saas_connection_config.id, + "fides_key": "base_test_saas", + "dataset": { + "fides_key": "mandrill_test", + "name": "Saas Dataset", + "description": "Example test dataset config", + "dataset_type": "mandrill", + "collections": [], + }, + }, + ) + yield dataset_config + dataset_config.delete(db) diff --git a/tests/ops/fixtures/privacy_center_config/bad_test_config.json b/tests/ops/fixtures/privacy_center_config/bad_test_config.json new file mode 100644 index 0000000000..2c84fb4239 --- /dev/null +++ b/tests/ops/fixtures/privacy_center_config/bad_test_config.json @@ -0,0 +1,51 @@ +{ + "title": "Privacy Center", + "description": "When you use our services, you’re trusting us with your information. We understand this is a big responsibility and work hard to protect your information and put you in control.", + "server_url_development": "http://localhost:8080/api/v1", + "server_url_production": "http://localhost:8080/api/v1", + "logo_path": "/logo.svg", + "actions": [], + "includeConsent": true, + "consent": { + "icon_path": "/consent.svg", + "title": "Manage your consent", + "description": "Manage your consent preferences, including the option to select 'Do Not Sell My Personal Information'.", + "identity_inputs": { + "email": "required", + "phone": "optional" + }, + "policy_key": "default_consent_policy", + "consent_options": [ + { + "fidesDataUseKey": "advertising", + "name": "Data Sales or Sharing", + "description": "We may use some of your personal information for behavioral advertising purposes, which may be interpreted as 'Data Sales' or 'Data Sharing' under regulations such as CCPA, CPRA, VCDPA, etc.", + "url": "https://example.com/privacy#data-sales", + "default": true, + "highlight": false, + "cookieKeys": ["data_sales"], + "executable": false + }, + { + "fidesDataUseKey": "advertising.first_party", + "name": "Email Marketing", + "description": "We may use some of your personal information to contact you about our products & services.", + "url": "https://example.com/privacy#email-marketing", + "default": true, + "highlight": false, + "cookieKeys": [], + "executable": true + }, + { + "fidesDataUseKey": "improve", + "name": "Product Analytics", + "description": "We may use some of your personal information to collect analytics about how you use our products & services.", + "url": "https://example.com/privacy#analytics", + "default": true, + "highlight": false, + "cookieKeys": [], + "executable": false + } + ] + } +} diff --git a/tests/ops/fixtures/privacy_center_config/test_config.json b/tests/ops/fixtures/privacy_center_config/test_config.json new file mode 100644 index 0000000000..22b1c131b0 --- /dev/null +++ b/tests/ops/fixtures/privacy_center_config/test_config.json @@ -0,0 +1,51 @@ +{ + "title": "Privacy Center", + "description": "When you use our services, you’re trusting us with your information. We understand this is a big responsibility and work hard to protect your information and put you in control.", + "server_url_development": "http://localhost:8080/api/v1", + "server_url_production": "http://localhost:8080/api/v1", + "logo_path": "/logo.svg", + "actions": [], + "includeConsent": true, + "consent": { + "icon_path": "/consent.svg", + "title": "Manage your consent", + "description": "Manage your consent preferences, including the option to select 'Do Not Sell My Personal Information'.", + "identity_inputs": { + "email": "required", + "phone": "optional" + }, + "policy_key": "default_consent_policy", + "consentOptions": [ + { + "fidesDataUseKey": "advertising", + "name": "Data Sales or Sharing", + "description": "We may use some of your personal information for behavioral advertising purposes, which may be interpreted as 'Data Sales' or 'Data Sharing' under regulations such as CCPA, CPRA, VCDPA, etc.", + "url": "https://example.com/privacy#data-sales", + "default": true, + "highlight": false, + "cookieKeys": ["data_sales"], + "executable": false + }, + { + "fidesDataUseKey": "advertising.first_party", + "name": "Email Marketing", + "description": "We may use some of your personal information to contact you about our products & services.", + "url": "https://example.com/privacy#email-marketing", + "default": true, + "highlight": false, + "cookieKeys": [], + "executable": true + }, + { + "fidesDataUseKey": "improve", + "name": "Product Analytics", + "description": "We may use some of your personal information to collect analytics about how you use our products & services.", + "url": "https://example.com/privacy#analytics", + "default": true, + "highlight": false, + "cookieKeys": [], + "executable": false + } + ] + } +} diff --git a/tests/ops/integration_tests/test_consent_request.py b/tests/ops/integration_tests/test_consent_request.py new file mode 100644 index 0000000000..7afc0f0369 --- /dev/null +++ b/tests/ops/integration_tests/test_consent_request.py @@ -0,0 +1,63 @@ +from uuid import uuid4 + +import pytest + +from fides.api.ops.models.policy import ActionType +from fides.api.ops.models.privacy_request import ( + ExecutionLog, + ExecutionLogStatus, + PrivacyRequest, +) +from fides.api.ops.service.privacy_request.request_runner_service import ( + build_consent_dataset_graph, +) +from fides.api.ops.task import graph_task + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_consent_request_task( + db, + consent_policy, + base_saas_connection_config, + base_saas_dataset_config, +) -> None: + + privacy_request = PrivacyRequest( + id=str(uuid4()), + consent_preferences=[{"data_use": "advertising", "opt_in": False}], + ) + + v = await graph_task.run_consent_request( + privacy_request, + consent_policy, + build_consent_dataset_graph([base_saas_dataset_config]), + [base_saas_connection_config], + {"email": "customer-1@example.com"}, + db, + ) + + assert v == { + "mandrill_test:mandrill_test": True + }, "graph has one node, and mocked request completed successfully" + + execution_logs = db.query(ExecutionLog).filter_by( + privacy_request_id=privacy_request.id + ) + + mandrill_logs = execution_logs.filter_by(collection_name="mandrill_test").order_by( + "created_at" + ) + assert mandrill_logs.count() == 2 + + assert [log.status for log in mandrill_logs] == [ + ExecutionLogStatus.in_processing, + ExecutionLogStatus.complete, + ] + + for log in mandrill_logs: + assert log.dataset_name == "mandrill_test" + assert ( + log.collection_name == "mandrill_test" + ), "Node-level is given the same name as the dataset name" + assert log.action_type == ActionType.consent diff --git a/tests/ops/models/test_consent_request.py b/tests/ops/models/test_consent_request.py new file mode 100644 index 0000000000..11b4d1719b --- /dev/null +++ b/tests/ops/models/test_consent_request.py @@ -0,0 +1,194 @@ +from datetime import datetime, timedelta, timezone +from time import sleep +from typing import List +from unittest import mock +from uuid import uuid4 + +import pytest + +from fides.api.ctl.database.seed import DEFAULT_CONSENT_POLICY +from fides.api.ops.api.v1.endpoints.consent_request_endpoints import ( + CONFIG_JSON_PATH, + load_executable_consent_options, + queue_privacy_request_to_propagate_consent, +) +from fides.api.ops.graph.config import CollectionAddress +from fides.api.ops.models.privacy_request import ( + Consent, + ConsentRequest, + PrivacyRequestStatus, + ProvidedIdentity, +) +from fides.api.ops.schemas.policy import PolicyResponse +from fides.api.ops.schemas.privacy_request import ( + BulkPostPrivacyRequests, + ConsentPreferences, + PrivacyRequestResponse, +) +from fides.core.config import get_config + +paused_location = CollectionAddress("test_dataset", "test_collection") + +CONFIG = get_config() + + +def test_consent(db): + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "encrypted_value": {"value": "test@email.com"}, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + + consent_data_1 = { + "provided_identity_id": provided_identity.id, + "data_use": "user.biometric_health", + "opt_in": True, + } + consent_1 = Consent.create(db, data=consent_data_1) + + consent_data_2 = { + "provided_identity_id": provided_identity.id, + "data_use": "user.browsing_history", + "opt_in": False, + } + consent_2 = Consent.create(db, data=consent_data_2) + data_uses = [x.data_use for x in provided_identity.consent] + + assert consent_data_1["data_use"] in data_uses + assert consent_data_2["data_use"] in data_uses + + provided_identity.delete(db) + + assert Consent.get(db, object_id=consent_1.id) is None + assert Consent.get(db, object_id=consent_2.id) is None + + +def test_consent_request(db): + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "encrypted_value": {"value": "test@email.com"}, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + + consent_request_1 = { + "provided_identity_id": provided_identity.id, + } + consent_1 = ConsentRequest.create(db, data=consent_request_1) + + consent_request_2 = { + "provided_identity_id": provided_identity.id, + } + consent_2 = ConsentRequest.create(db, data=consent_request_2) + + assert consent_1.provided_identity_id in provided_identity.id + assert consent_2.provided_identity_id in provided_identity.id + + provided_identity.delete(db) + + assert Consent.get(db, object_id=consent_1.id) is None + assert Consent.get(db, object_id=consent_2.id) is None + + +class TestLoadExecutableConsentOptionsHelper: + def test_load_executable_consent_options(self): + options: List[str] = load_executable_consent_options(CONFIG_JSON_PATH) + assert options == ["advertising", "advertising.first_party", "improve"] + + def test_load_options_some_not_executable(self): + other_options: List[str] = load_executable_consent_options( + "tests/ops/fixtures/privacy_center_config/test_config.json" + ) + assert other_options == ["advertising.first_party"] + + def test_load_invalid_config_json(self): + other_options: List[str] = load_executable_consent_options( + "tests/ops/fixtures/privacy_center_config/bad_test_config.json" + ) + assert other_options == [] + + def test_config_json_not_found(self): + with pytest.raises(FileNotFoundError): + load_executable_consent_options("bad_path.json") + + +class TestQueuePrivacyRequestToPropagateConsentHelper: + @mock.patch( + "fides.api.ops.api.v1.endpoints.consent_request_endpoints.create_privacy_request_func" + ) + def test_queue_privacy_request_to_propagate_consent( + self, mock_create_privacy_request, db, consent_policy + ): + mock_create_privacy_request.return_value = BulkPostPrivacyRequests( + succeeded=[ + PrivacyRequestResponse( + id="fake_privacy_request_id", + status=PrivacyRequestStatus.pending, + policy=PolicyResponse.from_orm(consent_policy), + ) + ], + failed=[], + ) + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "encrypted_value": {"value": "test@email.com"}, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + + consent_preferences = ConsentPreferences( + consent=[{"data_use": "advertising", "opt_in": False}] + ) + + queue_privacy_request_to_propagate_consent( + db=db, + provided_identity=provided_identity, + policy=DEFAULT_CONSENT_POLICY, + consent_preferences=consent_preferences, + ) + + assert mock_create_privacy_request.called + call_kwargs = mock_create_privacy_request.call_args[1] + assert call_kwargs["db"] == db + assert call_kwargs["data"][0].identity.email == "test@email.com" + assert len(call_kwargs["data"][0].consent_preferences) == 1 + assert call_kwargs["data"][0].consent_preferences[0].data_use == "advertising" + assert call_kwargs["data"][0].consent_preferences[0].opt_in is False + assert ( + call_kwargs["authenticated"] is True + ), "We already validated identity with a verification code earlier in the request" + + provided_identity.delete(mock_create_privacy_request) + + @mock.patch( + "fides.api.ops.api.v1.endpoints.consent_request_endpoints.create_privacy_request_func" + ) + def test_do_not_queue_privacy_request_if_no_executable_preferences( + self, mock_create_privacy_request, db, consent_policy + ): + mock_create_privacy_request.return_value = BulkPostPrivacyRequests( + succeeded=[ + PrivacyRequestResponse( + id="fake_privacy_request_id", + status=PrivacyRequestStatus.pending, + policy=PolicyResponse.from_orm(consent_policy), + ) + ], + failed=[], + ) + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "encrypted_value": {"value": "test@email.com"}, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + + queue_privacy_request_to_propagate_consent( + db=db, + provided_identity=provided_identity, + policy=DEFAULT_CONSENT_POLICY, + consent_preferences=ConsentPreferences(consent=[]), + ) + + assert not mock_create_privacy_request.called diff --git a/tests/ops/models/test_policy.py b/tests/ops/models/test_policy.py index b3cf980aa3..5df7b78570 100644 --- a/tests/ops/models/test_policy.py +++ b/tests/ops/models/test_policy.py @@ -150,24 +150,6 @@ def test_create_access_rule_with_no_storage_destination_is_invalid( assert exc.value.args[0] == "Access Rules must have a storage destination." -def test_consent_action_is_unsupported( - db: Session, - policy: Policy, -) -> None: - with pytest.raises(RuleValidationError) as exc: - Rule.create( - db=db, - data={ - "action_type": ActionType.consent.value, - "client_id": policy.client_id, - "name": "Invalid Rule", - "policy_id": policy.id, - "storage_destination_id": policy.rules[0].storage_destination.id, - }, - ) - assert exc.value.args[0] == "consent Rules are not supported at this time." - - def test_update_action_is_unsupported( db: Session, policy: Policy, diff --git a/tests/ops/models/test_privacy_request.py b/tests/ops/models/test_privacy_request.py index 160805952e..c65356d996 100644 --- a/tests/ops/models/test_privacy_request.py +++ b/tests/ops/models/test_privacy_request.py @@ -18,13 +18,10 @@ from fides.api.ops.models.policy import CurrentStep, Policy from fides.api.ops.models.privacy_request import ( CheckpointActionRequired, - Consent, - ConsentRequest, PrivacyRequest, PrivacyRequestError, PrivacyRequestNotifications, PrivacyRequestStatus, - ProvidedIdentity, can_run_checkpoint, ) from fides.api.ops.schemas.redis_cache import Identity @@ -770,65 +767,6 @@ def test_can_run_if_no_saved_checkpoint(self): ) -def test_consent(db): - provided_identity_data = { - "privacy_request_id": None, - "field_name": "email", - "encrypted_value": {"value": "test@email.com"}, - } - provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) - - consent_data_1 = { - "provided_identity_id": provided_identity.id, - "data_use": "user.biometric_health", - "opt_in": True, - } - consent_1 = Consent.create(db, data=consent_data_1) - - consent_data_2 = { - "provided_identity_id": provided_identity.id, - "data_use": "user.browsing_history", - "opt_in": False, - } - consent_2 = Consent.create(db, data=consent_data_2) - data_uses = [x.data_use for x in provided_identity.consent] - - assert consent_data_1["data_use"] in data_uses - assert consent_data_2["data_use"] in data_uses - - provided_identity.delete(db) - - assert Consent.get(db, object_id=consent_1.id) is None - assert Consent.get(db, object_id=consent_2.id) is None - - -def test_consent_request(db): - provided_identity_data = { - "privacy_request_id": None, - "field_name": "email", - "encrypted_value": {"value": "test@email.com"}, - } - provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) - - consent_request_1 = { - "provided_identity_id": provided_identity.id, - } - consent_1 = ConsentRequest.create(db, data=consent_request_1) - - consent_request_2 = { - "provided_identity_id": provided_identity.id, - } - consent_2 = ConsentRequest.create(db, data=consent_request_2) - - assert consent_1.provided_identity_id in provided_identity.id - assert consent_2.provided_identity_id in provided_identity.id - - provided_identity.delete(db) - - assert Consent.get(db, object_id=consent_1.id) is None - assert Consent.get(db, object_id=consent_2.id) is None - - def test_privacy_request_error_notification(db, policy): PrivacyRequestNotifications.create( db=db, diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index c6a633fd5e..7e2892bac2 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -14,6 +14,7 @@ ClientUnsuccessfulException, PrivacyRequestPaused, ) +from fides.api.ops.graph.graph import DatasetGraph from fides.api.ops.models.connectionconfig import AccessLevel from fides.api.ops.models.messaging import MessagingConfig from fides.api.ops.models.policy import CurrentStep, PolicyPostWebhook @@ -51,6 +52,7 @@ HmacMaskingStrategy, ) from fides.api.ops.service.privacy_request.request_runner_service import ( + build_consent_dataset_graph, run_webhooks_and_report_status, upload_access_results, ) @@ -2212,3 +2214,22 @@ def test_pass_on_empty_confirmed_input( assert mock_upload.call_args.kwargs["data"] == { "manual_webhook_example": [{"email": None, "last_name": None}] } + + +def test_build_consent_dataset_graph( + base_saas_dataset_config, + postgres_example_test_dataset_config_read_access, + mysql_example_test_dataset_config, +): + """Subject to change: currently returns a DatasetGraph made up from resources of saas-type""" + dataset_graph: DatasetGraph = build_consent_dataset_graph( + [ + base_saas_dataset_config, + postgres_example_test_dataset_config_read_access, + mysql_example_test_dataset_config, + ] + ) + assert len(dataset_graph.nodes.keys()) == 1 + assert [col_addr.value for col_addr in dataset_graph.nodes.keys()] == [ + "mandrill_test:mandrill_test" + ]