From ff5c7ca6f3c17f78976aec43e6468a9728b818b0 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 10 Jul 2023 17:59:03 -0500 Subject: [PATCH 1/9] Add two new tables, ServedNoticeHistory and LastServedNotice which share many fields with PrivacyPreferenceHistory and CurrentPrivacyPreference. These tables track every time a notice was served in the frontend for a given user, and then the last time a notice was served for a user. - Update GET /privacy-experience to declare on the notice if it was served to an end user or if an older version has been served --- .../7c562441c589_track_notices_served.py | 356 ++++++++++++ src/fides/api/db/base.py | 2 + src/fides/api/models/privacy_experience.py | 67 ++- src/fides/api/models/privacy_notice.py | 8 +- src/fides/api/models/privacy_preference.py | 532 +++++++++++++----- src/fides/api/schemas/privacy_notice.py | 6 + src/fides/cli/options.py | 92 ++- tests/fixtures/application_fixtures.py | 23 +- .../test_privacy_experience_endpoints.py | 28 +- tests/ops/api/v1/endpoints/test_system.py | 30 +- tests/ops/models/test_privacy_preference.py | 185 ++++++ 11 files changed, 1133 insertions(+), 196 deletions(-) create mode 100644 src/fides/api/alembic/migrations/versions/7c562441c589_track_notices_served.py diff --git a/src/fides/api/alembic/migrations/versions/7c562441c589_track_notices_served.py b/src/fides/api/alembic/migrations/versions/7c562441c589_track_notices_served.py new file mode 100644 index 0000000000..86183bcaaa --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/7c562441c589_track_notices_served.py @@ -0,0 +1,356 @@ +"""track notices served + +Revision ID: 7c562441c589 +Revises: 7315b9d7fda6 +Create Date: 2023-07-10 16:42:37.433995 + +""" +import sqlalchemy as sa +import sqlalchemy_utils +from alembic import op + +# revision identifiers, used by Alembic. +revision = "7c562441c589" +down_revision = "7315b9d7fda6" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "servednoticehistory", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "anonymized_ip_address", + sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), + nullable=True, + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "email", + sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), + nullable=True, + ), + sa.Column( + "fides_user_device", + sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), + nullable=True, + ), + sa.Column("hashed_email", sa.String(), nullable=True), + sa.Column("hashed_fides_user_device", sa.String(), nullable=True), + sa.Column("hashed_phone_number", sa.String(), nullable=True), + sa.Column( + "phone_number", + sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), + nullable=True, + ), + sa.Column("request_origin", sa.String(), nullable=True), + sa.Column("url_recorded", sa.String(), nullable=True), + sa.Column( + "user_agent", + sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), + nullable=True, + ), + sa.Column("user_geography", sa.String(), nullable=True), + sa.Column("acknowledge_mode", sa.Boolean(), nullable=True), + sa.Column("serving_component", sa.String(), nullable=False), + sa.Column("fides_user_device_provided_identity_id", sa.String(), nullable=True), + sa.Column("privacy_experience_config_history_id", sa.String(), nullable=True), + sa.Column("privacy_experience_id", sa.String(), nullable=True), + sa.Column("privacy_notice_history_id", sa.String(), nullable=False), + sa.Column("provided_identity_id", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["fides_user_device_provided_identity_id"], + ["providedidentity.id"], + ), + sa.ForeignKeyConstraint( + ["privacy_experience_config_history_id"], + ["privacyexperienceconfighistory.id"], + ), + sa.ForeignKeyConstraint( + ["privacy_experience_id"], + ["privacyexperience.id"], + ), + sa.ForeignKeyConstraint( + ["privacy_notice_history_id"], + ["privacynoticehistory.id"], + ), + sa.ForeignKeyConstraint( + ["provided_identity_id"], + ["providedidentity.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_servednoticehistory_created_at"), + "servednoticehistory", + ["created_at"], + unique=False, + ) + op.create_index( + op.f("ix_servednoticehistory_fides_user_device_provided_identity_id"), + "servednoticehistory", + ["fides_user_device_provided_identity_id"], + unique=False, + ) + op.create_index( + op.f("ix_servednoticehistory_hashed_email"), + "servednoticehistory", + ["hashed_email"], + unique=False, + ) + op.create_index( + op.f("ix_servednoticehistory_hashed_fides_user_device"), + "servednoticehistory", + ["hashed_fides_user_device"], + unique=False, + ) + op.create_index( + op.f("ix_servednoticehistory_hashed_phone_number"), + "servednoticehistory", + ["hashed_phone_number"], + unique=False, + ) + op.create_index( + op.f("ix_servednoticehistory_id"), "servednoticehistory", ["id"], unique=False + ) + op.create_index( + op.f("ix_servednoticehistory_privacy_experience_config_history_id"), + "servednoticehistory", + ["privacy_experience_config_history_id"], + unique=False, + ) + op.create_index( + op.f("ix_servednoticehistory_privacy_experience_id"), + "servednoticehistory", + ["privacy_experience_id"], + unique=False, + ) + op.create_index( + op.f("ix_servednoticehistory_privacy_notice_history_id"), + "servednoticehistory", + ["privacy_notice_history_id"], + unique=False, + ) + op.create_index( + op.f("ix_servednoticehistory_provided_identity_id"), + "servednoticehistory", + ["provided_identity_id"], + unique=False, + ) + op.create_index( + op.f("ix_servednoticehistory_serving_component"), + "servednoticehistory", + ["serving_component"], + unique=False, + ) + op.create_index( + op.f("ix_servednoticehistory_user_geography"), + "servednoticehistory", + ["user_geography"], + unique=False, + ) + op.create_table( + "lastservednotice", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("served_notice_history_id", sa.String(), nullable=False), + sa.Column("provided_identity_id", sa.String(), nullable=True), + sa.Column("fides_user_device_provided_identity_id", sa.String(), nullable=True), + sa.Column("privacy_notice_id", sa.String(), nullable=False), + sa.Column("privacy_notice_history_id", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["fides_user_device_provided_identity_id"], + ["providedidentity.id"], + ), + sa.ForeignKeyConstraint( + ["privacy_notice_history_id"], + ["privacynoticehistory.id"], + ), + sa.ForeignKeyConstraint( + ["privacy_notice_id"], + ["privacynotice.id"], + ), + sa.ForeignKeyConstraint( + ["provided_identity_id"], + ["providedidentity.id"], + ), + sa.ForeignKeyConstraint( + ["served_notice_history_id"], + ["servednoticehistory.id"], + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "fides_user_device_provided_identity_id", + "privacy_notice_id", + name="last_served_fides_user_device_identity_privacy_notice", + ), + sa.UniqueConstraint( + "provided_identity_id", + "privacy_notice_id", + name="last_served_identity_privacy_notice", + ), + ) + op.create_index( + op.f("ix_lastservednotice_created_at"), + "lastservednotice", + ["created_at"], + unique=False, + ) + op.create_index( + op.f("ix_lastservednotice_fides_user_device_provided_identity_id"), + "lastservednotice", + ["fides_user_device_provided_identity_id"], + unique=False, + ) + op.create_index( + op.f("ix_lastservednotice_id"), "lastservednotice", ["id"], unique=False + ) + op.create_index( + op.f("ix_lastservednotice_privacy_notice_history_id"), + "lastservednotice", + ["privacy_notice_history_id"], + unique=False, + ) + op.create_index( + op.f("ix_lastservednotice_privacy_notice_id"), + "lastservednotice", + ["privacy_notice_id"], + unique=False, + ) + op.create_index( + op.f("ix_lastservednotice_provided_identity_id"), + "lastservednotice", + ["provided_identity_id"], + unique=False, + ) + op.create_index( + op.f("ix_lastservednotice_served_notice_history_id"), + "lastservednotice", + ["served_notice_history_id"], + unique=False, + ) + op.create_index( + op.f("ix_lastservednotice_updated_at"), + "lastservednotice", + ["updated_at"], + unique=False, + ) + + op.add_column( + "privacypreferencehistory", + sa.Column("served_notice_history_id", sa.String(), nullable=True), + ) + op.create_index( + op.f("ix_privacypreferencehistory_served_notice_history_id"), + "privacypreferencehistory", + ["served_notice_history_id"], + unique=False, + ) + op.create_foreign_key( + "privacy_preference_served_notice_fk", + "privacypreferencehistory", + "servednoticehistory", + ["served_notice_history_id"], + ["id"], + ) + + +def downgrade(): + op.drop_constraint( + "privacy_preference_served_notice_fk", + "privacypreferencehistory", + type_="foreignkey", + ) + op.drop_index( + op.f("ix_privacypreferencehistory_served_notice_history_id"), + table_name="privacypreferencehistory", + ) + op.drop_column("privacypreferencehistory", "served_notice_history_id") + op.drop_index(op.f("ix_lastservednotice_updated_at"), table_name="lastservednotice") + op.drop_index( + op.f("ix_lastservednotice_served_notice_history_id"), + table_name="lastservednotice", + ) + op.drop_index( + op.f("ix_lastservednotice_provided_identity_id"), table_name="lastservednotice" + ) + op.drop_index( + op.f("ix_lastservednotice_privacy_notice_id"), table_name="lastservednotice" + ) + op.drop_index( + op.f("ix_lastservednotice_privacy_notice_history_id"), + table_name="lastservednotice", + ) + op.drop_index(op.f("ix_lastservednotice_id"), table_name="lastservednotice") + op.drop_index( + op.f("ix_lastservednotice_fides_user_device_provided_identity_id"), + table_name="lastservednotice", + ) + op.drop_index(op.f("ix_lastservednotice_created_at"), table_name="lastservednotice") + op.drop_table("lastservednotice") + op.drop_index( + op.f("ix_servednoticehistory_user_geography"), table_name="servednoticehistory" + ) + op.drop_index( + op.f("ix_servednoticehistory_serving_component"), + table_name="servednoticehistory", + ) + op.drop_index( + op.f("ix_servednoticehistory_provided_identity_id"), + table_name="servednoticehistory", + ) + op.drop_index( + op.f("ix_servednoticehistory_privacy_notice_history_id"), + table_name="servednoticehistory", + ) + op.drop_index( + op.f("ix_servednoticehistory_privacy_experience_id"), + table_name="servednoticehistory", + ) + op.drop_index( + op.f("ix_servednoticehistory_privacy_experience_config_history_id"), + table_name="servednoticehistory", + ) + op.drop_index(op.f("ix_servednoticehistory_id"), table_name="servednoticehistory") + op.drop_index( + op.f("ix_servednoticehistory_hashed_phone_number"), + table_name="servednoticehistory", + ) + op.drop_index( + op.f("ix_servednoticehistory_hashed_fides_user_device"), + table_name="servednoticehistory", + ) + op.drop_index( + op.f("ix_servednoticehistory_hashed_email"), table_name="servednoticehistory" + ) + op.drop_index( + op.f("ix_servednoticehistory_fides_user_device_provided_identity_id"), + table_name="servednoticehistory", + ) + op.drop_index( + op.f("ix_servednoticehistory_created_at"), table_name="servednoticehistory" + ) + op.drop_table("servednoticehistory") diff --git a/src/fides/api/db/base.py b/src/fides/api/db/base.py index 2d9b5c17ed..534c8a94cf 100644 --- a/src/fides/api/db/base.py +++ b/src/fides/api/db/base.py @@ -26,7 +26,9 @@ ) from fides.api.models.privacy_preference import ( CurrentPrivacyPreference, + LastServedNotice, PrivacyPreferenceHistory, + ServedNoticeHistory, ) from fides.api.models.privacy_request import PrivacyRequest from fides.api.models.registration import UserRegistration diff --git a/src/fides/api/models/privacy_experience.py b/src/fides/api/models/privacy_experience.py index 61a392d274..be46e78382 100644 --- a/src/fides/api/models/privacy_experience.py +++ b/src/fides/api/models/privacy_experience.py @@ -17,7 +17,10 @@ create_historical_data_from_record, update_if_modified, ) -from fides.api.models.privacy_preference import CurrentPrivacyPreference +from fides.api.models.privacy_preference import ( + CurrentPrivacyPreference, + LastServedNotice, +) from fides.api.models.privacy_request import ProvidedIdentity from fides.api.models.sql_models import System # type: ignore[attr-defined] @@ -273,21 +276,16 @@ def get_related_privacy_notices( notices: List[PrivacyNotice] = [] for notice in privacy_notice_query.order_by(PrivacyNotice.created_at.desc()): - saved_preference: Optional[ - CurrentPrivacyPreference - ] = CurrentPrivacyPreference.get_preference_for_notice_and_fides_user_device( + cache_saved_preference_on_notice( db=db, + notice=notice, + fides_user_provided_identity=fides_user_provided_identity, + ) + cache_notice_served( + db=db, + notice=notice, fides_user_provided_identity=fides_user_provided_identity, - privacy_notice=notice, ) - if saved_preference: - # Temporarily cache the preference for the given fides user device id in memory. - if saved_preference.preference_matches_latest_version: - notice.current_preference = saved_preference.preference - notice.outdated_preference = None - else: - notice.current_preference = None - notice.outdated_preference = saved_preference.preference notices.append(notice) return notices @@ -539,3 +537,46 @@ def upsert_privacy_experiences_after_config_update( ) linked_regions.append(region) return linked_regions, unlinked_regions + + +def cache_saved_preference_on_notice( + db: Session, notice: PrivacyNotice, fides_user_provided_identity: ProvidedIdentity +) -> None: + """At runtime, cache any saved preference values on the privacy notice""" + saved_preference: Optional[ + CurrentPrivacyPreference + ] = CurrentPrivacyPreference.get_preference_for_notice_and_fides_user_device( + db=db, + fides_user_provided_identity=fides_user_provided_identity, + privacy_notice=notice, + ) + if saved_preference: + # Temporarily cache the preference for the given fides user device id in memory. + if saved_preference.preference_matches_latest_version: + notice.current_preference = saved_preference.preference + notice.outdated_preference = None + else: + notice.current_preference = None + notice.outdated_preference = saved_preference.preference + + +def cache_notice_served( + db: Session, notice: PrivacyNotice, fides_user_provided_identity: ProvidedIdentity +) -> None: + """At runtime, cache if the current notice was served to the user, or a previous version + of the notice, if applicable""" + served_notice: Optional[ + LastServedNotice + ] = LastServedNotice.get_last_served_for_notice_and_fides_user_device( + db=db, + fides_user_provided_identity=fides_user_provided_identity, + privacy_notice=notice, + ) + if served_notice: + # Temporarily cache the preference for the given fides user device id in memory. + if served_notice.served_latest_version: + notice.current_served = True + notice.outdated_served = None + else: + notice.current_served = None + notice.outdated_served = True diff --git a/src/fides/api/models/privacy_notice.py b/src/fides/api/models/privacy_notice.py index e2ca09d41f..4948668d89 100644 --- a/src/fides/api/models/privacy_notice.py +++ b/src/fides/api/models/privacy_notice.py @@ -164,8 +164,12 @@ class PrivacyNoticeBase: # Attribute that can be temporarily cached as the result of "get_related_privacy_notices" # for a given user, for surfacing CurrentPrivacyPreferences for the user. - current_preference = None - outdated_preference = None + current_preference: Optional[UserConsentPreference] = None + outdated_preference: Optional[UserConsentPreference] = None + # Attributes that can be temporarily cached on the notice to see if the most + # recent version or a previous version of a notice have ever been served to the user + current_served: Optional[bool] = None + outdated_served: Optional[bool] = None def applies_to_system(self, system: System) -> bool: """Privacy Notice applies to System if a data use matches or the Privacy Notice diff --git a/src/fides/api/models/privacy_preference.py b/src/fides/api/models/privacy_preference.py index 7c9d3b787c..a868e93f86 100644 --- a/src/fides/api/models/privacy_preference.py +++ b/src/fides/api/models/privacy_preference.py @@ -1,14 +1,15 @@ -# pylint: disable=R0401, C0302 +# pylint: disable=R0401, C0302, W0143 from __future__ import annotations from enum import Enum -from typing import Any, Dict, Optional, Tuple, Type +from typing import Any, Dict, Optional, Tuple, Type, Union -from sqlalchemy import ARRAY, Column, DateTime +from sqlalchemy import ARRAY, Boolean, Column, DateTime from sqlalchemy import Enum as EnumColumn from sqlalchemy import ForeignKey, String, UniqueConstraint, func from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.mutable import MutableDict, MutableList from sqlalchemy.orm import Session, relationship from sqlalchemy_utils import StringEncryptedType @@ -44,16 +45,12 @@ class ConsentMethod(Enum): individual_notice = "individual_notice" -class PrivacyPreferenceHistory(Base): - """The DB ORM model for storing PrivacyPreferenceHistory, used for saving - every time consent preferences are saved for reporting purposes. +class ConsentReportingMixin: + """Mixin to be shared between PrivacyPreferenceHistory and ServedNoticeHistory + Contains common user details, and information about the notice, experience, experience config, etc. + for use in consent reporting. """ - # Systems capable of propagating their consent, and their status. If the preference is - # not relevant for the system, or we couldn't propagate a preference, the status is skipped - affected_system_status = Column( - MutableDict.as_mutable(JSONB), server_default="{}", default=dict - ) anonymized_ip_address = Column( StringEncryptedType( type_in=String(), @@ -81,18 +78,18 @@ class PrivacyPreferenceHistory(Base): padding="pkcs5", ), ) + # Optional FK to a fides user device id provided identity, if applicable - fides_user_device_provided_identity_id = Column( - String, ForeignKey(ProvidedIdentity.id), index=True - ) + @declared_attr + def fides_user_device_provided_identity_id(cls) -> Column: + return Column(String, ForeignKey(ProvidedIdentity.id), index=True) + # Hashed email, for searching hashed_email = Column(String, index=True) # Hashed fides user device id, for searching hashed_fides_user_device = Column(String, index=True) # Hashed phone number, for searching hashed_phone_number = Column(String, index=True) - # Button, individual notices - method = Column(EnumColumn(ConsentMethod)) # Encrypted phone number, for reporting phone_number = Column( StringEncryptedType( @@ -102,36 +99,201 @@ class PrivacyPreferenceHistory(Base): padding="pkcs5", ), ) - # Whether the user wants to opt in, opt out, or has acknowledged the notice - preference = Column(EnumColumn(UserConsentPreference), nullable=False, index=True) + # The specific version of the experience config the user was shown to present the relevant notice # Contains the version, language, button labels, description, etc. - privacy_experience_config_history_id = Column( - String, - ForeignKey("privacyexperienceconfighistory.id"), - nullable=True, - index=True, - ) + @declared_attr + def privacy_experience_config_history_id(cls) -> Column: + return Column( + String, + ForeignKey("privacyexperienceconfighistory.id"), + nullable=True, + index=True, + ) + # The specific experience under which the user was presented the relevant notice # Minimal information stored here, mostly just region and component type - privacy_experience_id = Column( - String, ForeignKey("privacyexperience.id"), nullable=True, index=True - ) + @declared_attr + def privacy_experience_id(cls) -> Column: + return Column( + String, ForeignKey("privacyexperience.id"), nullable=True, index=True + ) + # The specific historical record the user consented to - privacy_notice_history_id = Column( - String, ForeignKey(PrivacyNoticeHistory.id), nullable=False, index=True + @declared_attr + def privacy_notice_history_id(cls) -> Column: + return Column( + String, ForeignKey(PrivacyNoticeHistory.id), nullable=False, index=True + ) + + # Optional FK to a verified provided identity (like email or phone), if applicable + @declared_attr + def provided_identity_id(cls) -> Column: + return Column(String, ForeignKey(ProvidedIdentity.id), index=True) + + # Location where we received the request + request_origin = Column(EnumColumn(RequestOrigin)) # privacy center, overlay, API + + url_recorded = Column(String) + user_agent = Column( + StringEncryptedType( + type_in=String(), + key=CONFIG.security.app_encryption_key, + engine=AesGcmEngine, + padding="pkcs5", + ), ) + + user_geography = Column(String, index=True) + + # Relationships + @declared_attr + def privacy_notice_history(cls) -> relationship: + return relationship(PrivacyNoticeHistory) + + @declared_attr + def provided_identity(cls) -> relationship: + return relationship(ProvidedIdentity, foreign_keys=[cls.provided_identity_id]) + + @declared_attr + def fides_user_device_provided_identity(cls) -> relationship: + return relationship( + ProvidedIdentity, foreign_keys=[cls.fides_user_device_provided_identity_id] + ) + + +class ServingComponent(Enum): + overlay = "overlay" + banner = "banner" + privacy_center = "privacy_center" + + +def _validate_notice_and_identity( + db: Session, data: dict[str, Any] +) -> PrivacyNoticeHistory: + """Validates that the PrivacyNoticeHistory specified in the data dictionary + exists and that at least one provided identity type is supplied in the data + + Shares some common checks we run before saving PrivacyPreferenceHistory + or ServedPreferenceHistory + """ + privacy_notice_history = PrivacyNoticeHistory.get( + db=db, object_id=data.get("privacy_notice_history_id") + ) + if not privacy_notice_history: + raise PrivacyNoticeHistoryNotFound() + + if not data.get("provided_identity_id") and not data.get( + "fides_user_device_provided_identity_id" + ): + raise IdentityNotFoundException( + "Must supply a verified provided identity id or a fides_user_device_provided_identity_id" + ) + + return privacy_notice_history + + +class ServedNoticeHistory(ConsentReportingMixin, Base): + """A historical record of every time a notice was served in the UI to an end user""" + + acknowledge_mode = Column( + Boolean, + default=False, + ) + serving_component = Column(EnumColumn(ServingComponent), nullable=False, index=True) + + last_served_notice = ( + relationship( # Only exists if this is the same as the Last Served Notice + "LastServedNotice", + back_populates="served_notice_history", + cascade="all, delete", + uselist=False, + ) + ) + + @classmethod + def create( + cls: Type[ServedNoticeHistory], + db: Session, + *, + data: dict[str, Any], + check_name: bool = False, + ) -> ServedNoticeHistory: + """Method that creates a ServedNoticeHistory record and upserts a LastServedNotice record. + + The only difference between this and ServedNoticeHistory.save_notice_served_and_last_notice_served + is the response. + """ + history, _ = cls.save_notice_served_and_last_notice_served( + db, data=data, check_name=check_name + ) + return history + + @classmethod + def save_notice_served_and_last_notice_served( + cls: Type[ServedNoticeHistory], + db: Session, + *, + data: dict[str, Any], + check_name: bool = False, + ) -> Tuple[ServedNoticeHistory, LastServedNotice]: + """Create a ServedNoticeHistory record and then upsert the LastServedNotice record. + + If separate LastServedNotice records exist for both a verified provided identity and a fides user device + id provided identity, consolidate when the notice was last served into a single record. + + There is only one LastServedNotice for each PrivacyNotice/ProvidedIdentity. + """ + privacy_notice_history = _validate_notice_and_identity(db, data) + + created_served_notice_history = super().create( + db=db, data=data, check_name=check_name + ) + + last_served_data = { + "provided_identity_id": created_served_notice_history.provided_identity_id, + "privacy_notice_id": privacy_notice_history.privacy_notice_id, + "privacy_notice_history_id": privacy_notice_history.id, + "served_notice_history_id": created_served_notice_history.id, + "fides_user_device_provided_identity_id": created_served_notice_history.fides_user_device_provided_identity_id, + } + + upserted_last_served_notice_record: LastServedNotice = upsert_last_saved_record( + db, + created_historical_record=created_served_notice_history, + current_record_class=LastServedNotice, + privacy_notice_history=privacy_notice_history, + current_record_data=last_served_data, + ) + + return created_served_notice_history, upserted_last_served_notice_record + + +class PrivacyPreferenceHistory(ConsentReportingMixin, Base): + """The DB ORM model for storing PrivacyPreferenceHistory, used for saving + every time consent preferences are saved for reporting purposes. + """ + + # Systems capable of propagating their consent, and their status. If the preference is + # not relevant for the system, or we couldn't propagate a preference, the status is skipped + affected_system_status = Column( + MutableDict.as_mutable(JSONB), server_default="{}", default=dict + ) + + # Button, individual notices + method = Column(EnumColumn(ConsentMethod)) + # Whether the user wants to opt in, opt out, or has acknowledged the notice + preference = Column(EnumColumn(UserConsentPreference), nullable=False, index=True) + # The privacy request created to propage the preferences privacy_request_id = Column( String, ForeignKey(PrivacyRequest.id, ondelete="SET NULL"), index=True ) - # Optional FK to a verified provided identity (like email or phone), if applicable - provided_identity_id = Column(String, ForeignKey(ProvidedIdentity.id), index=True) - # Systems whose data use match. This doesn't necessarily mean we propagate. Some may be intentionally skipped later. + # Systems whose data use match. This doesn't necessarily mean we propagate. + # Some may be intentionally skipped later. relevant_systems = Column(MutableList.as_mutable(ARRAY(String))) - # Location where we received the request - request_origin = Column(EnumColumn(RequestOrigin)) # privacy center, overlay, API + # Relevant identities are added to the report during request propagation secondary_user_ids = Column( MutableDict.as_mutable( @@ -144,27 +306,15 @@ class PrivacyPreferenceHistory(Base): ), ) # Cache secondary user ids (cookies, etc) if known for reporting purposes. - url_recorded = Column(String) - user_agent = Column( - StringEncryptedType( - type_in=String(), - key=CONFIG.security.app_encryption_key, - engine=AesGcmEngine, - padding="pkcs5", - ), + # The record of where we served the notice in the frontend, for conversion purposes + served_notice_history_id = Column( + String, ForeignKey(ServedNoticeHistory.id), index=True ) - user_geography = Column(String, index=True) - # Relationships - privacy_notice_history = relationship(PrivacyNoticeHistory) privacy_request = relationship(PrivacyRequest, backref="privacy_preferences") - provided_identity = relationship( - ProvidedIdentity, foreign_keys=[provided_identity_id] - ) - fides_user_device_provided_identity = relationship( - ProvidedIdentity, foreign_keys=[fides_user_device_provided_identity_id] - ) + served_notice_history = relationship(ServedNoticeHistory, backref="served_notices") + current_privacy_preference = relationship( # Only exists if this is the same as the Current Privacy Preference "CurrentPrivacyPreference", back_populates="privacy_preference_history", @@ -229,18 +379,7 @@ def create_history_and_upsert_current_preference( There is only one CurrentPrivacyPreference for each PrivacyNotice/ProvidedIdentity. """ - privacy_notice_history = PrivacyNoticeHistory.get( - db=db, object_id=data.get("privacy_notice_history_id") - ) - if not privacy_notice_history: - raise PrivacyNoticeHistoryNotFound() - - if not data.get("provided_identity_id") and not data.get( - "fides_user_device_provided_identity_id" - ): - raise IdentityNotFoundException( - "Must supply a verified provided identity id or a fides_user_device_provided_identity_id" - ) + privacy_notice_history = _validate_notice_and_identity(db, data) data["relevant_systems"] = privacy_notice_history.calculate_relevant_systems(db) created_privacy_preference_history = super().create( @@ -256,65 +395,18 @@ def create_history_and_upsert_current_preference( "fides_user_device_provided_identity_id": created_privacy_preference_history.fides_user_device_provided_identity_id, } - existing_current_preference_on_provided_identity = None - # Check if there are Current Privacy Preferences saved against the ProvidedIdentity - if created_privacy_preference_history.provided_identity_id: - existing_current_preference_on_provided_identity = ( - db.query(CurrentPrivacyPreference) - .filter( - CurrentPrivacyPreference.provided_identity_id - == created_privacy_preference_history.provided_identity_id, - CurrentPrivacyPreference.privacy_notice_id - == privacy_notice_history.privacy_notice_id, - ) - .first() - ) - - # Check if there are Current Privacy Preferences saved against the Fides User Device Id Provided Identity - existing_current_preference_on_fides_user_device_provided_identity = None - if created_privacy_preference_history.fides_user_device_provided_identity_id: - existing_current_preference_on_fides_user_device_provided_identity = ( - db.query(CurrentPrivacyPreference) - .filter( - CurrentPrivacyPreference.fides_user_device_provided_identity_id - == created_privacy_preference_history.fides_user_device_provided_identity_id, - CurrentPrivacyPreference.privacy_notice_id - == privacy_notice_history.privacy_notice_id, - ) - .first() - ) - - if ( - existing_current_preference_on_provided_identity - and existing_current_preference_on_fides_user_device_provided_identity - and existing_current_preference_on_provided_identity - != existing_current_preference_on_fides_user_device_provided_identity - ): - # If both exist and were saved separately, let's delete one so we can consolidate. - # Let's consider these identities as belonging to the same user, and can save their current preferences together. - existing_current_preference_on_fides_user_device_provided_identity.delete( - db - ) - - current_preference: Optional[CurrentPrivacyPreference] = ( - existing_current_preference_on_provided_identity - or existing_current_preference_on_fides_user_device_provided_identity + current_preference: CurrentPrivacyPreference = upsert_last_saved_record( + db, + created_historical_record=created_privacy_preference_history, + current_record_class=CurrentPrivacyPreference, + privacy_notice_history=privacy_notice_history, + current_record_data=current_privacy_preference_data, ) - if current_preference: - current_preference.update(db=db, data=current_privacy_preference_data) - else: - current_preference = CurrentPrivacyPreference.create( - db=db, data=current_privacy_preference_data, check_name=False - ) - return created_privacy_preference_history, current_preference -class CurrentPrivacyPreference(Base): - """Stores only the user's most recently saved preference for a given privacy notice - - The specific privacy notice history and privacy preference history record are linked as well. - """ +class LastSavedMixin: + """Stores common fields for the last saved preference or last served notice""" created_at = Column(DateTime(timezone=True), server_default=func.now(), index=True) updated_at = Column( @@ -323,43 +415,72 @@ class CurrentPrivacyPreference(Base): onupdate=func.now(), index=True, ) + + @declared_attr + def provided_identity_id(cls) -> Column: + return Column(String, ForeignKey(ProvidedIdentity.id), index=True) + + @declared_attr + def fides_user_device_provided_identity_id(cls) -> Column: + return Column(String, ForeignKey(ProvidedIdentity.id), index=True) + + @declared_attr + def privacy_notice_id(cls) -> Column: + return Column(String, ForeignKey(PrivacyNotice.id), nullable=False, index=True) + + @declared_attr + def privacy_notice_history_id(cls) -> Column: + return Column( + String, ForeignKey(PrivacyNoticeHistory.id), nullable=False, index=True + ) + + # Relationships + @declared_attr + def provided_identity(cls) -> relationship: + return relationship(ProvidedIdentity, foreign_keys=[cls.provided_identity_id]) + + @declared_attr + def fides_user_device_provided_identity(cls) -> relationship: + return relationship( + ProvidedIdentity, foreign_keys=[cls.fides_user_device_provided_identity_id] + ) + + @declared_attr + def privacy_notice(cls) -> relationship: + return relationship(PrivacyNotice) + + @declared_attr + def privacy_notice_history(cls) -> relationship: + return relationship(PrivacyNoticeHistory) + + +class CurrentPrivacyPreference(LastSavedMixin, Base): + """Stores only the user's most recently saved preference for a given privacy notice + + The specific privacy notice history and privacy preference history record are linked as well. + """ + preference = Column(EnumColumn(UserConsentPreference), nullable=False, index=True) - provided_identity_id = Column(String, ForeignKey(ProvidedIdentity.id), index=True) - fides_user_device_provided_identity_id = Column( - String, ForeignKey(ProvidedIdentity.id), index=True - ) - privacy_notice_id = Column( - String, ForeignKey(PrivacyNotice.id), nullable=False, index=True - ) - privacy_notice_history_id = Column( - String, ForeignKey(PrivacyNoticeHistory.id), nullable=False, index=True - ) + privacy_preference_history_id = Column( String, ForeignKey(PrivacyPreferenceHistory.id), nullable=False, index=True ) - UniqueConstraint( - provided_identity_id, privacy_notice_id, name="identity_privacy_notice" - ) - - UniqueConstraint( - fides_user_device_provided_identity_id, - privacy_notice_id, - name="fides_user_device_identity_privacy_notice", + __table_args__ = ( + UniqueConstraint( + "provided_identity_id", "privacy_notice_id", name="identity_privacy_notice" + ), + UniqueConstraint( + "fides_user_device_provided_identity_id", + "privacy_notice_id", + name="fides_user_device_identity_privacy_notice", + ), ) # Relationships - privacy_notice = relationship(PrivacyNotice) - privacy_notice_history = relationship(PrivacyNoticeHistory) privacy_preference_history = relationship( PrivacyPreferenceHistory, cascade="delete, delete-orphan", single_parent=True ) - provided_identity = relationship( - ProvidedIdentity, foreign_keys=[provided_identity_id] - ) - fides_user_device_provided_identity = relationship( - ProvidedIdentity, foreign_keys=[fides_user_device_provided_identity_id] - ) @property def preference_matches_latest_version(self) -> bool: @@ -388,3 +509,126 @@ def get_preference_for_notice_and_fides_user_device( ) .first() ) + + +class LastServedNotice(LastSavedMixin, Base): + """Stores the last time a notice was served for a given user + + Also consolidates serving notices among various user identities. + """ + + served_notice_history_id = Column( + String, ForeignKey(ServedNoticeHistory.id), nullable=False, index=True + ) + + __table_args__ = ( + UniqueConstraint( + "provided_identity_id", + "privacy_notice_id", + name="last_served_identity_privacy_notice", + ), + UniqueConstraint( + "fides_user_device_provided_identity_id", + "privacy_notice_id", + name="last_served_fides_user_device_identity_privacy_notice", + ), + ) + + # Relationships + served_notice_history = relationship( + ServedNoticeHistory, cascade="delete, delete-orphan", single_parent=True + ) + + @property + def served_latest_version(self) -> bool: + """Returns True if the user was last served the latest version of this Notice""" + return ( + self.privacy_notice.privacy_notice_history_id + == self.privacy_notice_history_id + ) + + @classmethod + def get_last_served_for_notice_and_fides_user_device( + cls, + db: Session, + fides_user_provided_identity: ProvidedIdentity, + privacy_notice: PrivacyNotice, + ) -> Optional[LastServedNotice]: + """Retrieves the LastServedNotice record for the user with the given identity + for the given notice""" + return ( + db.query(LastServedNotice) + .filter( + LastServedNotice.fides_user_device_provided_identity_id + == fides_user_provided_identity.id, + LastServedNotice.privacy_notice_id == privacy_notice.id, + ) + .first() + ) + + +def upsert_last_saved_record( + db: Session, + created_historical_record: Union[PrivacyPreferenceHistory, ServedNoticeHistory], + current_record_class: Union[Type[CurrentPrivacyPreference], Type[LastServedNotice]], + privacy_notice_history: PrivacyNoticeHistory, + current_record_data: Dict[str, str], +) -> Union[CurrentPrivacyPreference, LastServedNotice]: + """ + Upserts our record of when a user was served consent or when the last saved their preferences for a given notice. + + After we save historical records, this method is called to update a separate table that stores just our most + recently saved record. + + If we have historical records for multiple identities, and we determine they are the same identity, + their preferences are consolidated into the same record. + """ + existing_record_for_provided_identity = None + # Check if we have "current" records for the ProvidedIdentity (usu an email or phone)/Privacy Notice + if created_historical_record.provided_identity_id: + existing_record_for_provided_identity = ( + db.query(current_record_class) + .filter( + current_record_class.provided_identity_id + == created_historical_record.provided_identity_id, + current_record_class.privacy_notice_id + == privacy_notice_history.privacy_notice_id, + ) + .first() + ) + + # Check if we have "current" records for the Fides User Device ID and the Privacy Notice + existing_record_for_fides_user_device = None + if created_historical_record.fides_user_device_provided_identity_id: + existing_record_for_fides_user_device = ( + db.query(current_record_class) + .filter( + current_record_class.fides_user_device_provided_identity_id + == created_historical_record.fides_user_device_provided_identity_id, + current_record_class.privacy_notice_id + == privacy_notice_history.privacy_notice_id, + ) + .first() + ) + + if ( + existing_record_for_provided_identity + and existing_record_for_fides_user_device + and existing_record_for_provided_identity + != existing_record_for_fides_user_device + ): + # If both exist and were saved separately, let's delete one so we can consolidate. + # Let's consider these identities as belonging to the same user. + existing_record_for_fides_user_device.delete(db) + + current_record: Optional[Union[CurrentPrivacyPreference, LastServedNotice]] = ( + existing_record_for_provided_identity or existing_record_for_fides_user_device + ) + if current_record: + current_record.update(db=db, data=current_record_data) + else: + current_record = current_record_class.create( + db=db, data=current_record_data, check_name=False + ) + + return current_record diff --git a/src/fides/api/schemas/privacy_notice.py b/src/fides/api/schemas/privacy_notice.py index ae8a229356..6ff69e98d8 100644 --- a/src/fides/api/schemas/privacy_notice.py +++ b/src/fides/api/schemas/privacy_notice.py @@ -157,6 +157,12 @@ class PrivacyNoticeResponseWithUserPreferences(PrivacyNoticeResponse): outdated_preference: Optional[ UserConsentPreference ] # If no current preference, check if we have a preference saved for a previous version. + current_served: Optional[ + bool + ] # Do we have a record of the most recent version of this notice being served to the user? + outdated_served: Optional[ + bool + ] # Have we served an older version of this notice to the user? class PrivacyNoticeHistorySchema(PrivacyNoticeCreation, PrivacyNoticeWithId): diff --git a/src/fides/cli/options.py b/src/fides/cli/options.py index 9ace71b1c4..013e4ecea2 100644 --- a/src/fides/cli/options.py +++ b/src/fides/cli/options.py @@ -21,7 +21,9 @@ def coverage_threshold_option(command: Callable) -> Callable: type=click.IntRange(0, 100), default=100, help="Set the coverage percentage for a passing scan.", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -30,7 +32,9 @@ def resource_type_argument(command: Callable) -> Callable: command = click.argument( "resource_type", type=click.Choice(model_list, case_sensitive=False), - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -39,7 +43,9 @@ def fides_key_argument(command: Callable) -> Callable: command = click.argument( "fides_key", type=str, - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -49,7 +55,9 @@ def fides_key_option(command: Callable) -> Callable: "-k", "--fides-key", default="", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -57,7 +65,9 @@ def manifests_dir_argument(command: Callable) -> Callable: "Add the manifests_dir argument." command = click.argument( "manifests_dir", type=click.Path(exists=True), default=".fides/" - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -65,7 +75,9 @@ def dry_flag(command: Callable) -> Callable: "Add a flag that prevents side-effects." command = click.option( "--dry", is_flag=True, help="Do not upload results to the Fides webserver." - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -76,7 +88,9 @@ def yes_flag(command: Callable) -> Callable: "-y", is_flag=True, help="Automatically responds `yes` to any prompts.", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -87,7 +101,9 @@ def verbose_flag(command: Callable) -> Callable: "-v", is_flag=True, help="Enable verbose output.", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -97,7 +113,9 @@ def include_null_flag(command: Callable) -> Callable: "--include-null", is_flag=True, help="Include null attributes.", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -109,7 +127,9 @@ def organization_fides_key_option(command: Callable) -> Callable: default="default_organization", show_default=True, help="The `organization_fides_key` of the `Organization` you want to specify.", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -121,7 +141,9 @@ def output_directory_option(command: Callable) -> Callable: default=".fides/", show_default=True, help="The output directory for the data map to be exported to.", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -131,7 +153,9 @@ def credentials_id_option(command: Callable) -> Callable: "--credentials-id", type=str, help="Use credentials keys defined within Fides config.", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -141,7 +165,9 @@ def connection_string_option(command: Callable) -> Callable: "--connection-string", type=str, help="Use the connection string option to connect to a database.", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -151,7 +177,9 @@ def okta_org_url_option(command: Callable) -> Callable: "--org-url", type=str, help="Connect to Okta using an 'Org URL'. _Requires options `--org-url` & `--token`._", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -161,7 +189,9 @@ def okta_token_option(command: Callable) -> Callable: "--token", type=str, help="Connect to Okta using a token. _Requires options `--org-url` and `--token`._", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -171,7 +201,9 @@ def aws_access_key_id_option(command: Callable) -> Callable: "--access_key_id", type=str, help="Connect to AWS using an `Access Key ID`. _Requires options `--access_key_id`, `--secret_access_key` & `--region`._", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -181,7 +213,9 @@ def aws_secret_access_key_option(command: Callable) -> Callable: "--secret_access_key", type=str, help="Connect to AWS using an `Access Key`. _Requires options `--access_key_id`, `--secret_access_key` & `--region`._", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -191,7 +225,9 @@ def aws_region_option(command: Callable) -> Callable: "--region", type=str, help="Connect to AWS using a specific `Region`. _Requires options `--access_key_id`, `--secret_access_key` & `--region`._", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -232,7 +268,9 @@ def username_option(command: Callable) -> Callable: help="If not provided, will be pulled from the config file or prompted for.", default="", callback=prompt_username, - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -243,7 +281,9 @@ def password_option(command: Callable) -> Callable: help="If not provided, will be pulled from the config file or prompted for.", default="", callback=prompt_password, - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -252,7 +292,9 @@ def first_name_option(command: Callable) -> Callable: "-f", "--first-name", default="", - )(command) # type: ignore + )( + command + ) # type: ignore return command @@ -261,15 +303,17 @@ def last_name_option(command: Callable) -> Callable: "-l", "--last-name", default="", - )(command) # type: ignore + )( + command + ) # type: ignore return command def username_argument(command: Callable) -> Callable: - command = click.argument("username", type=str)(command) # type: ignore + command = click.argument("username", type=str)(command) # type: ignore return command def password_argument(command: Callable) -> Callable: - command = click.argument("password", type=str)(command) # type: ignore + command = click.argument("password", type=str)(command) # type: ignore return command diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index ec706f37fb..5907a1b30c 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -47,7 +47,10 @@ PrivacyNotice, PrivacyNoticeRegion, ) -from fides.api.models.privacy_preference import PrivacyPreferenceHistory +from fides.api.models.privacy_preference import ( + PrivacyPreferenceHistory, + ServedNoticeHistory, +) from fides.api.models.privacy_request import ( Consent, ConsentRequest, @@ -1527,6 +1530,24 @@ def privacy_preference_history_us_ca_provide_for_fides_user( pref_1.delete(db) +@pytest.fixture(scope="function") +def served_notice_history_us_ca_provide_for_fides_user( + db: Session, privacy_notice_us_ca_provide, fides_user_provided_identity +) -> Generator: + pref_1 = ServedNoticeHistory.create( + db=db, + data={ + "acknowledge_mode": False, + "serving_component": "overlay", + "fides_user_device_provided_identity_id": fides_user_provided_identity.id, + "privacy_notice_history_id": privacy_notice_us_ca_provide.privacy_notice_history_id, + }, + check_name=False, + ) + yield pref_1 + pref_1.delete(db) + + @pytest.fixture(scope="function") def privacy_notice_us_co_third_party_sharing(db: Session) -> Generator: privacy_notice = PrivacyNotice.create( diff --git a/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py index cc9aba6e9e..d04d8e4417 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py @@ -549,16 +549,18 @@ def test_get_privacy_experiences_nonexistent_fides_user_device_id_filter( assert resp["privacy_notices"][0]["current_preference"] is None assert resp["privacy_notices"][0]["outdated_preference"] is None + assert resp["privacy_notices"][0]["current_served"] is None + assert resp["privacy_notices"][0]["outdated_served"] is None + @pytest.mark.usefixtures( "privacy_notice_us_ca_provide", "fides_user_provided_identity", "privacy_preference_history_us_ca_provide_for_fides_user", + "served_notice_history_us_ca_provide_for_fides_user", "privacy_experience_overlay", ) def test_get_privacy_experiences_fides_user_device_id_filter( - self, - api_client: TestClient, - url, + self, db, api_client: TestClient, url, privacy_notice_us_ca_provide ): resp = api_client.get( url + "?fides_user_device_id=051b219f-20e4-45df-82f7-5eb68a00889f", @@ -572,7 +574,27 @@ def test_get_privacy_experiences_fides_user_device_id_filter( assert data["privacy_notices"][0]["default_preference"] == "opt_out" assert data["privacy_notices"][0]["current_preference"] == "opt_in" assert data["privacy_notices"][0]["outdated_preference"] is None + # Assert that the notice was served is surfaced + assert data["privacy_notices"][0]["current_served"] is True + assert data["privacy_notices"][0]["outdated_served"] is None + assert ( data["privacy_notices"][0]["notice_key"] == "example_privacy_notice_us_ca_provide" ) + + privacy_notice_us_ca_provide.update(db, data={"description": "new_description"}) + assert privacy_notice_us_ca_provide.version == 2.0 + assert privacy_notice_us_ca_provide.description == "new_description" + resp = api_client.get( + url + "?fides_user_device_id=051b219f-20e4-45df-82f7-5eb68a00889f", + ) + assert resp.status_code == 200 + data = resp.json()["items"][0] + # Assert current preference is displayed for fides user device id + assert data["privacy_notices"][0]["consent_mechanism"] == "opt_in" + assert data["privacy_notices"][0]["default_preference"] == "opt_out" + assert data["privacy_notices"][0]["current_preference"] is None + assert data["privacy_notices"][0]["outdated_preference"] == "opt_in" + assert data["privacy_notices"][0]["current_served"] is None + assert data["privacy_notices"][0]["outdated_served"] is True diff --git a/tests/ops/api/v1/endpoints/test_system.py b/tests/ops/api/v1/endpoints/test_system.py index 6a28092c61..e56ee71cec 100644 --- a/tests/ops/api/v1/endpoints/test_system.py +++ b/tests/ops/api/v1/endpoints/test_system.py @@ -7,13 +7,12 @@ from sqlalchemy.orm import Session from starlette.status import ( HTTP_200_OK, + HTTP_204_NO_CONTENT, HTTP_401_UNAUTHORIZED, HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND, - HTTP_204_NO_CONTENT, ) from starlette.testclient import TestClient -from fides.common.api.v1.urn_registry import V1_URL_PREFIX from fides.api.models.connectionconfig import ( AccessLevel, @@ -27,12 +26,13 @@ from fides.api.models.sql_models import Dataset from fides.common.api.scope_registry import ( CONNECTION_CREATE_OR_UPDATE, + CONNECTION_DELETE, CONNECTION_READ, SAAS_CONNECTION_INSTANTIATE, STORAGE_DELETE, SYSTEM_MANAGER_UPDATE, - CONNECTION_DELETE, ) +from fides.common.api.v1.urn_registry import V1_URL_PREFIX from tests.conftest import generate_role_header_for_user from tests.fixtures.saas.connection_template_fixtures import instantiate_connector @@ -184,8 +184,12 @@ def test_patch_connections_role_check_viewer( if assign_system: assign_url = V1_URL_PREFIX + f"/user/{acting_user_role.id}/system-manager" - system_manager_auth_header = generate_auth_header(scopes=[SYSTEM_MANAGER_UPDATE]) - api_client.put(assign_url, headers=system_manager_auth_header, json=[system.fides_key]) + system_manager_auth_header = generate_auth_header( + scopes=[SYSTEM_MANAGER_UPDATE] + ) + api_client.put( + assign_url, headers=system_manager_auth_header, json=[system.fides_key] + ) auth_header = generate_system_manager_header([system.id]) else: auth_header = generate_role_header_for_user( @@ -293,8 +297,12 @@ def test_get_connection_configs_role_viewer( if assign_system: assign_url = V1_URL_PREFIX + f"/user/{acting_user_role.id}/system-manager" - system_manager_auth_header = generate_auth_header(scopes=[SYSTEM_MANAGER_UPDATE]) - api_client.put(assign_url, headers=system_manager_auth_header, json=[system.fides_key]) + system_manager_auth_header = generate_auth_header( + scopes=[SYSTEM_MANAGER_UPDATE] + ) + api_client.put( + assign_url, headers=system_manager_auth_header, json=[system.fides_key] + ) auth_header = generate_system_manager_header([system.id]) else: auth_header = generate_role_header_for_user( @@ -513,8 +521,12 @@ def test_delete_connection_configs_role_viewer( if assign_system: assign_url = V1_URL_PREFIX + f"/user/{acting_user_role.id}/system-manager" - system_manager_auth_header = generate_auth_header(scopes=[SYSTEM_MANAGER_UPDATE]) - api_client.put(assign_url, headers=system_manager_auth_header, json=[system.fides_key]) + system_manager_auth_header = generate_auth_header( + scopes=[SYSTEM_MANAGER_UPDATE] + ) + api_client.put( + assign_url, headers=system_manager_auth_header, json=[system.fides_key] + ) auth_header = generate_system_manager_header([system.id]) else: auth_header = generate_role_header_for_user( diff --git a/tests/ops/models/test_privacy_preference.py b/tests/ops/models/test_privacy_preference.py index a1cafff501..745b896ea4 100644 --- a/tests/ops/models/test_privacy_preference.py +++ b/tests/ops/models/test_privacy_preference.py @@ -9,9 +9,13 @@ IdentityNotFoundException, PrivacyNoticeHistoryNotFound, ) +from fides.api.models.privacy_experience import PrivacyExperienceConfig from fides.api.models.privacy_preference import ( + LastServedNotice, PrivacyPreferenceHistory, RequestOrigin, + ServedNoticeHistory, + ServingComponent, UserConsentPreference, ) from fides.api.models.privacy_request import ( @@ -641,3 +645,184 @@ def test_anonymize_ipv6(self): anonymize_ip_address("2001:0db8:85a3:0000:0000:8a2e:0370:7334") == "2001:0db8:85a3:0000:0000:0000:0000:0000" ) + + +class TestServedNoticeHistory: + def test_created_record_of_notice_served_and_upsert_last_served_notice( + self, + db, + privacy_notice, + privacy_experience_privacy_center, + experience_config_privacy_center, + ): + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "hashed_value": ProvidedIdentity.hash_value("ethyca@email.com"), + "encrypted_value": {"value": "ethyca@email.com"}, + } + fides_user_provided_identity_data = { + "privacy_request_id": None, + "field_name": "fides_user_device_id", + "hashed_value": ProvidedIdentity.hash_value( + "test_fides_user_device_id_abcdefg" + ), + "encrypted_value": {"value": "test_fides_user_device_id_abcdefg"}, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + fides_user_provided_identity = ProvidedIdentity.create( + db, data=fides_user_provided_identity_data + ) + + privacy_notice_history = privacy_notice.histories[0] + + email, hashed_email = extract_identity_from_provided_identity( + provided_identity, ProvidedIdentityType.email + ) + phone_number, hashed_phone_number = extract_identity_from_provided_identity( + provided_identity, ProvidedIdentityType.phone_number + ) + ( + fides_user_device_id, + hashed_device_id, + ) = extract_identity_from_provided_identity( + fides_user_provided_identity, ProvidedIdentityType.fides_user_device_id + ) + + served_notice_history_record = ServedNoticeHistory.create( + db=db, + data={ + "anonymized_ip_address": "12.214.31.0", + "email": email, + "fides_user_device": fides_user_device_id, + "fides_user_device_provided_identity_id": fides_user_provided_identity.id, + "hashed_email": hashed_email, + "hashed_fides_user_device": hashed_device_id, + "hashed_phone_number": hashed_phone_number, + "phone_number": phone_number, + "privacy_notice_history_id": privacy_notice_history.id, + "provided_identity_id": provided_identity.id, + "request_origin": "privacy_center", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/324.42 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/425.24", + "user_geography": "us_co", + "url_recorded": "example.com/privacy_center", + "acknowledge_mode": False, + "serving_component": ServingComponent.privacy_center, + "privacy_experience_id": privacy_experience_privacy_center.id, + "privacy_experience_config_history_id": experience_config_privacy_center.histories[ + 0 + ].id, + }, + check_name=False, + ) + assert served_notice_history_record.email == "ethyca@email.com" + assert ( + served_notice_history_record.hashed_email + == provided_identity.hashed_value + is not None + ) + assert ( + served_notice_history_record.fides_user_device + == fides_user_device_id + is not None + ) + assert ( + served_notice_history_record.hashed_fides_user_device + == hashed_device_id + is not None + ) + assert ( + served_notice_history_record.fides_user_device_provided_identity + == fides_user_provided_identity + ) + + assert served_notice_history_record.phone_number is None + assert served_notice_history_record.hashed_phone_number is None + assert ( + served_notice_history_record.privacy_notice_history + == privacy_notice_history + ) + assert served_notice_history_record.provided_identity == provided_identity + assert ( + served_notice_history_record.request_origin == RequestOrigin.privacy_center + ) + assert ( + served_notice_history_record.user_agent + == "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/324.42 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/425.24" + ) + assert served_notice_history_record.user_geography == "us_co" + assert served_notice_history_record.url_recorded == "example.com/privacy_center" + + # Assert ServedNoticeHistory record upserted + last_served_notice = served_notice_history_record.last_served_notice + assert last_served_notice.privacy_notice_history == privacy_notice_history + assert last_served_notice.privacy_notice_id == privacy_notice.id + assert last_served_notice.provided_identity_id == provided_identity.id + assert ( + last_served_notice.fides_user_device_provided_identity_id + == fides_user_provided_identity.id + ) + assert ( + last_served_notice.served_notice_history_id + == served_notice_history_record.id + ) + + served_notice_history_record.delete(db) + last_served_notice.delete(db) + + +class TestLastServedNotice: + def test_served_latest_version( + self, + db, + served_notice_history_us_ca_provide_for_fides_user, + privacy_notice_us_ca_provide, + ): + last_served = ( + served_notice_history_us_ca_provide_for_fides_user.last_served_notice + ) + assert last_served.served_latest_version is True + + privacy_notice_us_ca_provide.update(db, data={"description": "new_description"}) + assert privacy_notice_us_ca_provide.version == 2.0 + assert privacy_notice_us_ca_provide.description == "new_description" + + assert last_served.served_latest_version is False + + def test_get_last_served_for_notice_and_fides_user_device( + self, + db, + fides_user_provided_identity, + served_notice_history_us_ca_provide_for_fides_user, + privacy_notice_us_ca_provide, + empty_provided_identity, + privacy_notice, + ): + retrieved_record = ( + LastServedNotice.get_last_served_for_notice_and_fides_user_device( + db, + fides_user_provided_identity=fides_user_provided_identity, + privacy_notice=privacy_notice_us_ca_provide, + ) + ) + assert ( + retrieved_record + == served_notice_history_us_ca_provide_for_fides_user.last_served_notice + ) + + assert ( + LastServedNotice.get_last_served_for_notice_and_fides_user_device( + db, + fides_user_provided_identity=empty_provided_identity, + privacy_notice=privacy_notice_us_ca_provide, + ) + is None + ) + assert ( + LastServedNotice.get_last_served_for_notice_and_fides_user_device( + db, + fides_user_provided_identity=fides_user_provided_identity, + privacy_notice=privacy_notice, + ) + is None + ) From dfd460687c4aaa5558006cbd3f3d89918034d008 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 11 Jul 2023 11:15:59 -0500 Subject: [PATCH 2/9] Add a new public endpoint PATCH /notices-served to be called by the UI when notices are served. Multiple notices that were served together should be sent in the same request. --- .../endpoints/privacy_preference_endpoints.py | 155 +++++++++++++-- src/fides/api/schemas/privacy_preference.py | 38 +++- src/fides/common/api/v1/urn_registry.py | 1 + .../test_privacy_preference_endpoints.py | 182 ++++++++++++++++++ 4 files changed, 355 insertions(+), 21 deletions(-) diff --git a/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py index 22f386a93e..13a362b655 100644 --- a/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py @@ -1,6 +1,6 @@ import ipaddress from datetime import datetime -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Type, Union from fastapi import Depends, HTTPException, Request, Response from fastapi.params import Security @@ -29,7 +29,9 @@ ) from fides.api.models.privacy_preference import ( CurrentPrivacyPreference, + LastServedNotice, PrivacyPreferenceHistory, + ServedNoticeHistory, ) from fides.api.models.privacy_request import ( ConsentRequest, @@ -42,6 +44,9 @@ ConsentReportingSchema, CurrentPrivacyPreferenceReportingSchema, CurrentPrivacyPreferenceSchema, + LastServedNoticeSchema, + NoticesServedCreate, + NoticesServedRequest, PrivacyPreferencesCreate, PrivacyPreferencesRequest, ) @@ -65,6 +70,7 @@ CONSENT_REQUEST_PRIVACY_PREFERENCES_WITH_ID, CURRENT_PRIVACY_PREFERENCES_REPORT, HISTORICAL_PRIVACY_PREFERENCES_REPORT, + NOTICES_SERVED, PRIVACY_PREFERENCES, V1_URL_PREFIX, ) @@ -121,7 +127,7 @@ def consent_request_verify_for_privacy_preferences( def verify_privacy_notice_and_historical_records( - db: Session, data: PrivacyPreferencesRequest + db: Session, notice_history_list: List[str] ) -> None: """ Used when saving privacy preferences: runs a check that makes sure all the privacy notice histories referenced by @@ -138,18 +144,13 @@ def verify_privacy_notice_and_historical_records( PrivacyNoticeHistory.privacy_notice_id == PrivacyNotice.id, ) .filter( - PrivacyNoticeHistory.id.in_( - [ - consent_option.privacy_notice_history_id - for consent_option in data.preferences - ] - ), + PrivacyNoticeHistory.id.in_(notice_history_list), ) .distinct() .count() ) - if privacy_notice_count < len(data.preferences): + if privacy_notice_count < len(notice_history_list): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, detail="Invalid privacy notice histories in request", @@ -231,12 +232,14 @@ def _get_request_origin_and_config( return origin, experience_config_history_id -def supplement_privacy_preferences_with_user_and_experience_details( - db: Session, request: Request, data: PrivacyPreferencesRequest -) -> PrivacyPreferencesCreate: +def supplement_request_with_user_and_experience_details( + db: Session, + request: Request, + data: Union[PrivacyPreferencesRequest, NoticesServedRequest], + resource_type: Union[Type[PrivacyPreferencesCreate], Type[NoticesServedCreate]], +) -> Union[PrivacyPreferencesCreate, NoticesServedCreate]: """ Pull additional user information from request headers and experience to record for consent reporting purposes - """ request_headers = request.headers @@ -247,7 +250,7 @@ def supplement_privacy_preferences_with_user_and_experience_details( db, data ) - return PrivacyPreferencesCreate( + return resource_type( **data.dict(), anonymized_ip_address=anonymize_ip_address(ip_address), experience_config_history_id=experience_config_history_id, @@ -278,7 +281,13 @@ def save_privacy_preferences_with_verified_identity( Creates historical records for these preferences for record keeping, and also updates current preferences. Creates a privacy request to propagate preferences to third party systems. """ - verify_privacy_notice_and_historical_records(db=db, data=data) + verify_privacy_notice_and_historical_records( + db=db, + notice_history_list=[ + consent_option.privacy_notice_history_id + for consent_option in data.preferences + ], + ) consent_request, provided_identity = _get_consent_request_and_provided_identity( db=db, consent_request_id=consent_request_id, @@ -311,8 +320,8 @@ def save_privacy_preferences_with_verified_identity( consent_request=consent_request, verified_provided_identity=provided_identity, fides_user_provided_identity=fides_user_provided_identity, - request_data=supplement_privacy_preferences_with_user_and_experience_details( - db, request, data + request_data=supplement_request_with_user_and_experience_details( + db, request, data, resource_type=PrivacyPreferencesCreate ), ) @@ -429,6 +438,70 @@ def _save_privacy_preferences_for_identities( return upserted_current_preferences +def _save_notices_served_for_identities( + db: Session, + verified_provided_identity: Optional[ProvidedIdentity], + fides_user_provided_identity: Optional[ProvidedIdentity], + request_data: NoticesServedCreate, +) -> List[LastServedNotice]: + """ + Common code to that saves that notices have been served (both historical and current records) + """ + created_notices_served: List[ServedNoticeHistory] = [] + upserted_last_served: List[LastServedNotice] = [] + + email, hashed_email = extract_identity_from_provided_identity( + verified_provided_identity, ProvidedIdentityType.email + ) + phone_number, hashed_phone_number = extract_identity_from_provided_identity( + verified_provided_identity, ProvidedIdentityType.phone_number + ) + fides_user_device_id, hashed_device_id = extract_identity_from_provided_identity( + fides_user_provided_identity, ProvidedIdentityType.fides_user_device_id + ) + + for notice_history_id in request_data.privacy_notice_history_ids: + ( + historical_preference, + current_preference, + ) = ServedNoticeHistory.save_notice_served_and_last_notice_served( + db=db, + data={ + "acknowledge_mode": request_data.acknowledge_mode, + "anonymized_ip_address": request_data.anonymized_ip_address, + "email": email, + "fides_user_device": fides_user_device_id, + "fides_user_device_provided_identity_id": fides_user_provided_identity.id + if fides_user_provided_identity + else None, + "hashed_email": hashed_email, + "hashed_fides_user_device": hashed_device_id, + "hashed_phone_number": hashed_phone_number, + "phone_number": phone_number, + "privacy_experience_config_history_id": request_data.experience_config_history_id + if request_data.experience_config_history_id + else None, + "privacy_experience_id": request_data.privacy_experience_id + if request_data.privacy_experience_id + else None, + "privacy_notice_history_id": notice_history_id, + "provided_identity_id": verified_provided_identity.id + if verified_provided_identity + else None, + "request_origin": request_data.request_origin, + "serving_component": request_data.serving_component, + "url_recorded": request_data.url_recorded, + "user_agent": request_data.user_agent, + "user_geography": request_data.user_geography, + }, + check_name=False, + ) + created_notices_served.append(historical_preference) + upserted_last_served.append(current_preference) + + return upserted_last_served + + @router.patch( PRIVACY_PREFERENCES, status_code=HTTP_200_OK, @@ -447,7 +520,13 @@ def save_privacy_preferences( Creates historical records for these preferences for record keeping, and also updates current preferences. Creates a privacy request to propagate preferences to third party systems. """ - verify_privacy_notice_and_historical_records(db=db, data=data) + verify_privacy_notice_and_historical_records( + db=db, + notice_history_list=[ + consent_option.privacy_notice_history_id + for consent_option in data.preferences + ], + ) fides_user_provided_identity = get_or_create_fides_user_device_id_provided_identity( db=db, identity_data=data.browser_identity @@ -459,8 +538,8 @@ def save_privacy_preferences( consent_request=None, verified_provided_identity=None, fides_user_provided_identity=fides_user_provided_identity, - request_data=supplement_privacy_preferences_with_user_and_experience_details( - db, request, data + request_data=supplement_request_with_user_and_experience_details( + db, request, data, resource_type=PrivacyPreferencesCreate ), ) @@ -569,3 +648,39 @@ def get_historical_consent_report( query = query.order_by(PrivacyPreferenceHistory.created_at.desc()) return paginate(query, params) + + +@router.patch( + NOTICES_SERVED, + status_code=HTTP_200_OK, + response_model=List[LastServedNoticeSchema], +) +@fides_limiter.limit(CONFIG.security.public_request_rate_limit) +def save_notices_served( + *, + db: Session = Depends(get_db), + data: NoticesServedRequest, + request: Request, + response: Response, # required for rate limiting +) -> List[LastServedNotice]: + """Records what notices were served to a fides user device id. + + All notices that were served in an experience should be included in the request body. + """ + verify_privacy_notice_and_historical_records( + db=db, notice_history_list=data.privacy_notice_history_ids + ) + + fides_user_provided_identity = get_or_create_fides_user_device_id_provided_identity( + db=db, identity_data=data.browser_identity + ) + + logger.info("Recording notices served with respect to fides user device id") + return _save_notices_served_for_identities( + db=db, + verified_provided_identity=None, + fides_user_provided_identity=fides_user_provided_identity, + request_data=supplement_request_with_user_and_experience_details( + db, request, data, resource_type=NoticesServedCreate + ), + ) diff --git a/src/fides/api/schemas/privacy_preference.py b/src/fides/api/schemas/privacy_preference.py index 9d6a382594..767671ea2e 100644 --- a/src/fides/api/schemas/privacy_preference.py +++ b/src/fides/api/schemas/privacy_preference.py @@ -6,7 +6,11 @@ from fides.api.custom_types import SafeStr from fides.api.models.privacy_notice import UserConsentPreference -from fides.api.models.privacy_preference import ConsentMethod, RequestOrigin +from fides.api.models.privacy_preference import ( + ConsentMethod, + RequestOrigin, + ServingComponent, +) from fides.api.models.privacy_request import ExecutionLogStatus, PrivacyRequestStatus from fides.api.schemas.base_class import FidesSchema from fides.api.schemas.policy import ActionType @@ -51,6 +55,38 @@ class MinimalPrivacyPreferenceHistorySchema(FidesSchema): privacy_notice_history: PrivacyNoticeHistorySchema +class NoticesServedRequest(FidesSchema): + """Request body when indicating that notices were served in the UI""" + + browser_identity: Identity + code: Optional[SafeStr] + privacy_notice_history_ids: List[str] + privacy_experience_id: Optional[SafeStr] + user_geography: Optional[SafeStr] + acknowledge_mode: Optional[bool] + serving_component: ServingComponent + + +class NoticesServedCreate(NoticesServedRequest): + """Schema used on the backend only where we supplement the NoticesServedRequest requets body + with information obtained from the request headers and the experience""" + + anonymized_ip_address: Optional[str] + experience_config_history_id: Optional[SafeStr] + request_origin: Optional[RequestOrigin] + url_recorded: Optional[SafeStr] + user_agent: Optional[SafeStr] + + +class LastServedNoticeSchema(FidesSchema): + """Schema that surfaces the last version of a notice that was shown to a user""" + + id: str + updated_at: datetime + privacy_notice_history: PrivacyNoticeHistorySchema + served_notice_history_id: str + + class ConsentReportingSchema(FidesSchema): """Schema for consent reporting - largely a join of PrivacyPreferenceHistory and PrivacyRequest""" diff --git a/src/fides/common/api/v1/urn_registry.py b/src/fides/common/api/v1/urn_registry.py index 6ff88a2348..7094e4153e 100644 --- a/src/fides/common/api/v1/urn_registry.py +++ b/src/fides/common/api/v1/urn_registry.py @@ -25,6 +25,7 @@ "/consent-request/{consent_request_id}/verify-for-privacy-preferences" ) PRIVACY_PREFERENCES = "/privacy-preferences" +NOTICES_SERVED = "/notices-served" # Reporting endpoints - have records for *all* users HISTORICAL_PRIVACY_PREFERENCES_REPORT = "/historical-privacy-preferences" diff --git a/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py index 24d569d556..d709db41f4 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py @@ -10,8 +10,10 @@ from fides.api.models.privacy_preference import ( ConsentMethod, CurrentPrivacyPreference, + LastServedNotice, PrivacyPreferenceHistory, RequestOrigin, + ServingComponent, UserConsentPreference, ) from fides.api.models.privacy_request import ( @@ -31,6 +33,7 @@ CONSENT_REQUEST_PRIVACY_PREFERENCES_WITH_ID, CURRENT_PRIVACY_PREFERENCES_REPORT, HISTORICAL_PRIVACY_PREFERENCES_REPORT, + NOTICES_SERVED, PRIVACY_PREFERENCES, V1_URL_PREFIX, ) @@ -1697,3 +1700,182 @@ def test_get_current_preferences_date_filtering( assert response.status_code == 400 assert "Value specified for updated_lt" in response.json()["detail"] assert "must be after updated_gt" in response.json()["detail"] + + +class TestSaveNoticesServedForFidesDeviceId: + @pytest.fixture(scope="function") + def url(self) -> str: + return V1_URL_PREFIX + NOTICES_SERVED + + @pytest.fixture(scope="function") + def request_body(self, privacy_notice, privacy_experience_overlay): + return { + "browser_identity": { + "fides_user_device_id": "f7e54703-cd57-495e-866d-042e67c81734", + }, + "privacy_notice_history_ids": [privacy_notice.histories[0].id], + "privacy_experience_id": privacy_experience_overlay.id, + "user_geography": "us_ca", + "acknowledge_mode": False, + "serving_component": ServingComponent.banner.value, + } + + @pytest.mark.usefixtures( + "privacy_notice", + ) + def test_no_fides_user_device_id_supplied(self, api_client, url, request_body): + """We need a fides user device id in the request body to save that consent was served""" + del request_body["browser_identity"]["fides_user_device_id"] + response = api_client.patch( + url, json=request_body, headers={"Origin": "http://localhost:8080"} + ) + assert response.status_code == 422 + + @pytest.mark.usefixtures( + "privacy_notice", + ) + def test_bad_fides_user_device_id_supplied(self, api_client, url, request_body): + """Testing validation that fides user device id must be in expected uuid format""" + request_body["browser_identity"][ + "fides_user_device_id" + ] = "bad_fides_user_device_id" + response = api_client.patch( + url, json=request_body, headers={"Origin": "http://localhost:8080"} + ) + assert response.status_code == 422 + assert ( + response.json()["detail"][0]["msg"] + == "badly formed hexadecimal UUID string" + ) + + def test_record_notices_served_with_bad_notice(self, api_client, url, request_body): + """Every notice history in request body needs to be valid""" + request_body["privacy_notice_history_ids"] = ["bad_history"] + response = api_client.patch( + url, json=request_body, headers={"Origin": "http://localhost:8080"} + ) + assert response.status_code == 400 + + def test_record_notices_served_bad_experience_id( + self, + api_client, + url, + request_body, + ): + """Privacy experiences need to be valid when recording notices served""" + request_body["privacy_experience_id"] = "bad_id" + response = api_client.patch( + url, json=request_body, headers={"Origin": "http://localhost:8080"} + ) + assert response.status_code == 404 + assert response.json()["detail"] == f"Privacy Experience 'bad_id' not found." + + @mock.patch( + "fides.api.api.v1.endpoints.privacy_preference_endpoints.anonymize_ip_address" + ) + def test_record_notices_served_with_respect_to_fides_user_device_id( + self, + mock_anonymize, + db, + api_client, + url, + request_body, + privacy_notice, + privacy_experience_overlay, + ): + """Test recording that a notice was served to the given user with this fides user device id + + We create a ServedNoticeHistory record for every single time a notice is served. + Separately, we upsert a LastServedNotice record whose intent is to capture the last saved + notice across versions and across time, consolidating known user identities + + """ + test_device_id = "f7e54703-cd57-495e-866d-042e67c81734" + masked_ip = "12.214.31.0" + mock_anonymize.return_value = masked_ip + response = api_client.patch( + url, json=request_body, headers={"Origin": "http://localhost:8080"} + ) + assert response.status_code == 200 + assert len(response.json()) == 1 + response_json = response.json()[0] + assert ( + response_json["privacy_notice_history"]["id"] + == privacy_notice.histories[0].id + ) + + served_notice_history_id = response_json["served_notice_history_id"] + + # Fetch last served notice record that was updated + last_served_notice = LastServedNotice.get(db, object_id=response_json["id"]) + assert last_served_notice.created_at is not None + assert last_served_notice.updated_at is not None + assert last_served_notice.provided_identity_id is None + assert last_served_notice.fides_user_device_provided_identity_id is not None + assert last_served_notice.privacy_notice_id == privacy_notice.id + assert ( + last_served_notice.privacy_notice_history_id + == privacy_notice.histories[0].id + ) + + # Get corresponding historical record that was just created + served_notice_history = last_served_notice.served_notice_history + + assert served_notice_history.id == served_notice_history_id + assert served_notice_history.updated_at is not None + assert served_notice_history.anonymized_ip_address == masked_ip + assert served_notice_history.created_at is not None + assert served_notice_history.email is None + assert ( + served_notice_history.fides_user_device == test_device_id + ) # Cached here for reporting + assert served_notice_history.hashed_email is None + assert ( + served_notice_history.hashed_fides_user_device + == ProvidedIdentity.hash_value(test_device_id) + ) # Cached here for reporting + assert served_notice_history.hashed_phone_number is None + assert served_notice_history.phone_number is None + assert ( + served_notice_history.request_origin == RequestOrigin.overlay + ) # Retrieved from privacy experience history + assert served_notice_history.url_recorded is None + assert ( + served_notice_history.user_agent == "testclient" + ) # Retrieved from request headers + assert served_notice_history.user_geography == "us_ca" + assert served_notice_history.acknowledge_mode is False + assert served_notice_history.serving_component == ServingComponent.banner + + fides_user_device_provided_identity = ( + served_notice_history.fides_user_device_provided_identity + ) + # Same fides user device identity added to both the historical and current record + assert ( + last_served_notice.fides_user_device_provided_identity + == fides_user_device_provided_identity + ) + assert ( + fides_user_device_provided_identity.hashed_value + == ProvidedIdentity.hash_value(test_device_id) + ) + assert ( + fides_user_device_provided_identity.encrypted_value["value"] + == test_device_id + ) + + assert ( + served_notice_history.privacy_experience_config_history_id + == privacy_experience_overlay.experience_config.experience_config_history_id + ) + assert ( + served_notice_history.privacy_experience_id == privacy_experience_overlay.id + ) + assert ( + served_notice_history.privacy_notice_history_id + == privacy_notice.histories[0].id + ) + assert served_notice_history.provided_identity_id is None + + last_served_notice.delete(db) + served_notice_history.delete(db) From 498847a813e5c7088d2d1e9f1b25a9be2ce600e5 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 11 Jul 2023 12:59:39 -0500 Subject: [PATCH 3/9] Add a similar endpoint to save that notices were served in the privacy center. This endpoint has logic capable of handling multiple identity types, such as an email and a fides user device id. --- .../endpoints/privacy_preference_endpoints.py | 115 ++++- src/fides/api/schemas/privacy_preference.py | 4 +- src/fides/common/api/v1/urn_registry.py | 1 + .../test_privacy_preference_endpoints.py | 447 ++++++++++++++++++ 4 files changed, 545 insertions(+), 22 deletions(-) diff --git a/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py index 13a362b655..1fc5d9516d 100644 --- a/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py @@ -66,6 +66,7 @@ PRIVACY_PREFERENCE_HISTORY_READ, ) from fides.common.api.v1.urn_registry import ( + CONSENT_REQUEST_NOTICES_SERVED, CONSENT_REQUEST_PRIVACY_PREFERENCES_VERIFY, CONSENT_REQUEST_PRIVACY_PREFERENCES_WITH_ID, CURRENT_PRIVACY_PREFERENCES_REPORT, @@ -293,26 +294,15 @@ def save_privacy_preferences_with_verified_identity( consent_request_id=consent_request_id, verification_code=data.code, ) - if not provided_identity.hashed_value: - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, detail="Provided identity missing" - ) - if provided_identity.field_name == ProvidedIdentityType.fides_user_device_id: - # If consent request was saved against a fides user device id only, this is our primary identity - # This workflow is for when customers don't want to collect email/phone number. - fides_user_provided_identity = provided_identity - provided_identity = None # type: ignore[assignment] - else: - # Get the fides user device id from the dictionary of browser identifiers - try: - fides_user_provided_identity = ( - get_or_create_fides_user_device_id_provided_identity( - db=db, identity_data=data.browser_identity - ) - ) - except HTTPException: - fides_user_provided_identity = None + ( + provided_identity, + fides_user_provided_identity, + ) = classify_identities_for_privacy_center_consent_reporting( + db=db, + provided_identity=provided_identity, + browser_identity=data.browser_identity, + ) logger.info("Saving privacy preferences") return _save_privacy_preferences_for_identities( @@ -663,7 +653,8 @@ def save_notices_served( request: Request, response: Response, # required for rate limiting ) -> List[LastServedNotice]: - """Records what notices were served to a fides user device id. + """Records what notices were served to a fides user device id only. Generally called by the banner + or an overlay. All notices that were served in an experience should be included in the request body. """ @@ -684,3 +675,87 @@ def save_notices_served( db, request, data, resource_type=NoticesServedCreate ), ) + + +def classify_identities_for_privacy_center_consent_reporting( + db: Session, + provided_identity: ProvidedIdentity, + browser_identity: Optional[Identity], +) -> Tuple[Optional[ProvidedIdentity], Optional[ProvidedIdentity]]: + """For consent reporting purposes, we separate out the type of identity that identifies the user + + We want to classify the "provided_identity" as an identifier saved against an email or phone, + and the "fides_user_provided_identity" as an identifier saved against the fides user device id. + """ + if not provided_identity.hashed_value: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, detail="Provided identity missing" + ) + + if provided_identity.field_name == ProvidedIdentityType.fides_user_device_id: + # If consent request was saved against a fides user device id only, this is our primary identity + # This workflow is for when customers don't want to collect email/phone number. + fides_user_provided_identity = provided_identity + provided_identity = None # type: ignore[assignment] + else: + # Get the fides user device id from the dictionary of browser identifiers + try: + fides_user_provided_identity = ( + get_or_create_fides_user_device_id_provided_identity( + db=db, identity_data=browser_identity + ) + ) + except HTTPException: + fides_user_provided_identity = None + + return provided_identity, fides_user_provided_identity + + +@router.patch( + CONSENT_REQUEST_NOTICES_SERVED, + status_code=HTTP_200_OK, + response_model=List[LastServedNoticeSchema], +) +def save_notices_served_via_privacy_center( + *, + consent_request_id: str, + db: Session = Depends(get_db), + data: NoticesServedRequest, + request: Request, +) -> List[LastServedNotice]: + """Saves that notices were served via a verified identity flow (privacy center) + + Capable of saving that notices were served against a verified email/phone number and a fides user device id + simultaneously. + + Creates a ServedNoticeHistory history record for every notice in the request and upserts + a LastServedNotice record. + """ + verify_privacy_notice_and_historical_records( + db=db, + notice_history_list=data.privacy_notice_history_ids, + ) + _, provided_identity = _get_consent_request_and_provided_identity( + db=db, + consent_request_id=consent_request_id, + verification_code=data.code, + ) + + ( + provided_identity, + fides_user_provided_identity, + ) = classify_identities_for_privacy_center_consent_reporting( + db=db, + provided_identity=provided_identity, + browser_identity=data.browser_identity, + ) + + logger.info("Saving notices served for privacy center") + return _save_notices_served_for_identities( + db=db, + verified_provided_identity=provided_identity, + fides_user_provided_identity=fides_user_provided_identity, + request_data=supplement_request_with_user_and_experience_details( + db, request, data, resource_type=NoticesServedCreate + ), + ) diff --git a/src/fides/api/schemas/privacy_preference.py b/src/fides/api/schemas/privacy_preference.py index 767671ea2e..7297986c1f 100644 --- a/src/fides/api/schemas/privacy_preference.py +++ b/src/fides/api/schemas/privacy_preference.py @@ -59,8 +59,8 @@ class NoticesServedRequest(FidesSchema): """Request body when indicating that notices were served in the UI""" browser_identity: Identity - code: Optional[SafeStr] - privacy_notice_history_ids: List[str] + code: Optional[SafeStr] # For verified identity workflow only + privacy_notice_history_ids: List[SafeStr] privacy_experience_id: Optional[SafeStr] user_geography: Optional[SafeStr] acknowledge_mode: Optional[bool] diff --git a/src/fides/common/api/v1/urn_registry.py b/src/fides/common/api/v1/urn_registry.py index 7094e4153e..d2495161fa 100644 --- a/src/fides/common/api/v1/urn_registry.py +++ b/src/fides/common/api/v1/urn_registry.py @@ -21,6 +21,7 @@ CONSENT_REQUEST_PRIVACY_PREFERENCES_WITH_ID = ( "/consent-request/{consent_request_id}/privacy-preferences" ) +CONSENT_REQUEST_NOTICES_SERVED = "/consent-request/{consent_request_id}/notices-served" CONSENT_REQUEST_PRIVACY_PREFERENCES_VERIFY = ( "/consent-request/{consent_request_id}/verify-for-privacy-preferences" ) diff --git a/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py index d709db41f4..ae7e2c1854 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py @@ -29,6 +29,7 @@ PRIVACY_PREFERENCE_HISTORY_READ, ) from fides.common.api.v1.urn_registry import ( + CONSENT_REQUEST_NOTICES_SERVED, CONSENT_REQUEST_PRIVACY_PREFERENCES_VERIFY, CONSENT_REQUEST_PRIVACY_PREFERENCES_WITH_ID, CURRENT_PRIVACY_PREFERENCES_REPORT, @@ -1879,3 +1880,449 @@ def test_record_notices_served_with_respect_to_fides_user_device_id( last_served_notice.delete(db) served_notice_history.delete(db) + + +class TestSaveNoticesServedPrivacyCenter: + @pytest.fixture(scope="function") + def verification_code(self) -> str: + return "abcd" + + @pytest.fixture(scope="function") + def request_body( + self, privacy_notice, verification_code, privacy_experience_privacy_center + ): + return { + "browser_identity": { + "fides_user_device_id": "f7e54703-cd57-495e-866d-042e67c81734" + }, + "code": verification_code, + "privacy_notice_history_ids": [privacy_notice.histories[0].id], + "privacy_experience_id": privacy_experience_privacy_center.id, + "user_geography": "us_co", + "serving_component": ServingComponent.privacy_center.value, + } + + @pytest.mark.usefixtures( + "subject_identity_verification_required", + ) + def test_save_notices_served_no_matching_consent_request_id( + self, api_client, request_body + ): + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_NOTICES_SERVED.format(consent_request_id='non_existent_consent_id')}", + json=request_body, + ) + assert response.status_code == 404 + assert "not found" in response.json()["detail"] + + @pytest.mark.usefixtures( + "subject_identity_verification_required", + ) + def test_save_notices_served_code_expired( + self, provided_identity_and_consent_request, api_client, request_body + ): + _, consent_request = provided_identity_and_consent_request + + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_NOTICES_SERVED.format(consent_request_id=consent_request.id)}", + json=request_body, + ) + assert response.status_code == 400 + assert "code expired" in response.json()["detail"] + + @pytest.mark.usefixtures( + "subject_identity_verification_required", + ) + def test_save_notices_served_invalid_code( + self, + provided_identity_and_consent_request, + api_client, + verification_code, + request_body, + ): + _, consent_request = provided_identity_and_consent_request + consent_request.cache_identity_verification_code(verification_code) + + request_body["code"] = "non_matching_code" + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_NOTICES_SERVED.format(consent_request_id=consent_request.id)}", + json=request_body, + ) + assert response.status_code == 403 + assert "Incorrect identification" in response.json()["detail"] + + @pytest.mark.usefixtures("subject_identity_verification_required", "system") + @mock.patch( + "fides.api.api.v1.endpoints.privacy_preference_endpoints.anonymize_ip_address" + ) + def test_save_notices_served( + self, + mock_anonymize, + provided_identity_and_consent_request, + api_client, + verification_code, + db: Session, + request_body, + privacy_notice, + privacy_experience_privacy_center, + ): + """Verify code, save notices served, and return. + + The fact that notices were served is saved with respect to two provided identities - + one for the email and one for the fides user device id + """ + masked_ip = "12.214.31.0" # Mocking because hostname for FastAPI TestClient is "testclient" + mock_anonymize.return_value = masked_ip + + provided_identity, consent_request = provided_identity_and_consent_request + consent_request.cache_identity_verification_code(verification_code) + + test_device_id = "f7e54703-cd57-495e-866d-042e67c81734" + + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_NOTICES_SERVED.format(consent_request_id=consent_request.id)}", + json=request_body, + ) + assert response.status_code == 200 + assert len(response.json()) == 1 + response_json = response.json()[0] + + assert ( + response_json["privacy_notice_history"]["id"] + == privacy_notice.histories[0].id + ) + + served_notice_history_id = response_json["served_notice_history_id"] + + # Fetch last served notice record that was updated + last_served_notice = LastServedNotice.get(db, object_id=response_json["id"]) + assert last_served_notice.created_at is not None + assert last_served_notice.updated_at is not None + assert last_served_notice.provided_identity_id == provided_identity.id + assert last_served_notice.fides_user_device_provided_identity_id is not None + assert last_served_notice.privacy_notice_id == privacy_notice.id + assert ( + last_served_notice.privacy_notice_history_id + == privacy_notice.histories[0].id + ) + + # Get corresponding historical record that was just created + served_notice_history = last_served_notice.served_notice_history + + assert served_notice_history.id == served_notice_history_id + assert served_notice_history.updated_at is not None + assert served_notice_history.anonymized_ip_address == masked_ip + assert served_notice_history.created_at is not None + assert served_notice_history.email == "test@email.com" + assert ( + served_notice_history.fides_user_device == test_device_id + ) # Cached here for reporting + assert served_notice_history.hashed_email == ProvidedIdentity.hash_value( + "test@email.com" + ) + assert ( + served_notice_history.hashed_fides_user_device + == ProvidedIdentity.hash_value(test_device_id) + ) # Cached here for reporting + assert served_notice_history.hashed_phone_number is None + assert served_notice_history.phone_number is None + assert ( + served_notice_history.request_origin == RequestOrigin.privacy_center + ) # Retrieved from privacy experience history + assert served_notice_history.url_recorded is None + assert ( + served_notice_history.user_agent == "testclient" + ) # Retrieved from request headers + assert served_notice_history.user_geography == "us_co" + assert served_notice_history.acknowledge_mode is False + assert ( + served_notice_history.serving_component == ServingComponent.privacy_center + ) + + fides_user_device_provided_identity = ( + served_notice_history.fides_user_device_provided_identity + ) + # Same fides user device identity added to both the historical and current record + assert ( + last_served_notice.fides_user_device_provided_identity + == fides_user_device_provided_identity + ) + assert ( + fides_user_device_provided_identity.hashed_value + == ProvidedIdentity.hash_value(test_device_id) + ) + assert ( + fides_user_device_provided_identity.encrypted_value["value"] + == test_device_id + ) + + assert ( + served_notice_history.privacy_experience_config_history_id + == privacy_experience_privacy_center.experience_config.experience_config_history_id + ) + assert ( + served_notice_history.privacy_experience_id + == privacy_experience_privacy_center.id + ) + assert ( + served_notice_history.privacy_notice_history_id + == privacy_notice.histories[0].id + ) + assert served_notice_history.provided_identity_id == provided_identity.id + + last_served_notice.delete(db) + served_notice_history.delete(db) + + @pytest.mark.usefixtures("subject_identity_verification_required", "system") + @mock.patch( + "fides.api.api.v1.endpoints.privacy_preference_endpoints.anonymize_ip_address" + ) + def test_save_notices_served_device_id_only( + self, + mock_anonymize, + fides_user_provided_identity_and_consent_request, + api_client, + verification_code, + db: Session, + request_body, + privacy_notice, + privacy_experience_privacy_center, + ): + """Verify code, save notices served, and return. + + This tests when someone has set up their privacy center so we're not actually collecting + email/phone number there. The original consent request was saved against a fides user + device id only + """ + masked_ip = "12.214.31.0" # Mocking because hostname for FastAPI TestClient is "testclient" + mock_anonymize.return_value = masked_ip + + ( + fides_user_provided_identity, + consent_request, + ) = fides_user_provided_identity_and_consent_request + + consent_request.cache_identity_verification_code(verification_code) + + test_device_id = "051b219f-20e4-45df-82f7-5eb68a00889f" + + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_NOTICES_SERVED.format(consent_request_id=consent_request.id)}", + json=request_body, + ) + assert response.status_code == 200 + assert len(response.json()) == 1 + response_json = response.json()[0] + + assert ( + response_json["privacy_notice_history"]["id"] + == privacy_notice.histories[0].id + ) + + served_notice_history_id = response_json["served_notice_history_id"] + + # Fetch last served notice record that was updated + last_served_notice = LastServedNotice.get(db, object_id=response_json["id"]) + assert last_served_notice.created_at is not None + assert last_served_notice.updated_at is not None + assert last_served_notice.provided_identity_id is None + assert ( + last_served_notice.fides_user_device_provided_identity_id + == fides_user_provided_identity.id + ) + assert last_served_notice.privacy_notice_id == privacy_notice.id + assert ( + last_served_notice.privacy_notice_history_id + == privacy_notice.histories[0].id + ) + + # Get corresponding historical record that was just created + served_notice_history = last_served_notice.served_notice_history + + assert served_notice_history.id == served_notice_history_id + assert served_notice_history.updated_at is not None + assert served_notice_history.anonymized_ip_address == masked_ip + assert served_notice_history.created_at is not None + assert served_notice_history.email is None + assert ( + served_notice_history.fides_user_device == test_device_id + ) # Cached here for reporting + assert served_notice_history.hashed_email is None + assert ( + served_notice_history.hashed_fides_user_device + == ProvidedIdentity.hash_value(test_device_id) + ) # Cached here for reporting + assert served_notice_history.hashed_phone_number is None + assert served_notice_history.phone_number is None + assert ( + served_notice_history.request_origin == RequestOrigin.privacy_center + ) # Retrieved from privacy experience history + assert served_notice_history.url_recorded is None + assert ( + served_notice_history.user_agent == "testclient" + ) # Retrieved from request headers + assert served_notice_history.user_geography == "us_co" + assert served_notice_history.acknowledge_mode is False + assert ( + served_notice_history.serving_component == ServingComponent.privacy_center + ) + + fides_user_device_provided_identity = ( + served_notice_history.fides_user_device_provided_identity + ) + # Same fides user device identity added to both the historical and current record + assert ( + last_served_notice.fides_user_device_provided_identity + == fides_user_provided_identity + == fides_user_device_provided_identity + ) + assert ( + fides_user_device_provided_identity.hashed_value + == ProvidedIdentity.hash_value(test_device_id) + ) + assert ( + fides_user_device_provided_identity.encrypted_value["value"] + == test_device_id + ) + + assert ( + served_notice_history.privacy_experience_config_history_id + == privacy_experience_privacy_center.experience_config.experience_config_history_id + ) + assert ( + served_notice_history.privacy_experience_id + == privacy_experience_privacy_center.id + ) + assert ( + served_notice_history.privacy_notice_history_id + == privacy_notice.histories[0].id + ) + assert served_notice_history.provided_identity_id is None + + last_served_notice.delete(db) + served_notice_history.delete(db) + + @pytest.mark.usefixtures( + "subject_identity_verification_required", + ) + def test_save_notices_served_invalid_code_respects_attempt_count( + self, + provided_identity_and_consent_request, + api_client, + verification_code, + request_body, + ): + _, consent_request = provided_identity_and_consent_request + consent_request.cache_identity_verification_code(verification_code) + + request_body["code"] = "987632" # Bad code + + for _ in range(0, CONFIG.security.identity_verification_attempt_limit): + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_NOTICES_SERVED.format(consent_request_id=consent_request.id)}", + json=request_body, + ) + assert response.status_code == 403 + assert "Incorrect identification" in response.json()["detail"] + + assert ( + consent_request._get_cached_verification_code_attempt_count() + == CONFIG.security.identity_verification_attempt_limit + ) + + request_body["code"] = verification_code + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_NOTICES_SERVED.format(consent_request_id=consent_request.id)}", + json=request_body, + ) + assert response.status_code == 403 + assert ( + response.json()["detail"] == f"Attempt limit hit for '{consent_request.id}'" + ) + assert consent_request.get_cached_verification_code() is None + assert consent_request._get_cached_verification_code_attempt_count() == 0 + + @pytest.mark.usefixtures( + "subject_identity_verification_required", + ) + @patch("fides.api.models.privacy_request.ConsentRequest.verify_identity") + def test_save_notices_served_missing_identity_data( + self, + mock_verify_identity: MagicMock, + db, + api_client, + verification_code, + request_body, + ): + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "hashed_value": None, + "encrypted_value": None, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + + consent_request_data = { + "provided_identity_id": provided_identity.id, + } + consent_request = ConsentRequest.create(db, data=consent_request_data) + consent_request.cache_identity_verification_code(verification_code) + + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_NOTICES_SERVED.format(consent_request_id=consent_request.id)}", + json=request_body, + ) + + assert response.status_code == 404 + assert mock_verify_identity.called + assert "missing" in response.json()["detail"] + + @pytest.mark.usefixtures( + "subject_identity_verification_required", + ) + def test_save_privacy_notices_served_invalid_privacy_notice_history_id( + self, + provided_identity_and_consent_request, + api_client, + verification_code, + request_body, + ): + _, consent_request = provided_identity_and_consent_request + consent_request.cache_identity_verification_code(verification_code) + + request_body["privacy_notice_history_ids"] = ["bad_id"] + + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_NOTICES_SERVED.format(consent_request_id=consent_request.id)}", + json=request_body, + ) + assert ( + response.status_code == 400 + ), "Gets picked up by the duplicate privacy notice check" + + @pytest.mark.usefixtures( + "subject_identity_verification_required", + ) + def test_save_notices_viewed_for_the_same_notice_in_one_request( + self, + provided_identity_and_consent_request, + api_client, + verification_code, + request_body, + privacy_notice, + ): + _, consent_request = provided_identity_and_consent_request + consent_request.cache_identity_verification_code(verification_code) + + request_body["privacy_notice_history_ids"] = [ + privacy_notice.histories[0].id, + privacy_notice.histories[0].id, + ] + + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_NOTICES_SERVED.format(consent_request_id=consent_request.id)}", + json=request_body, + ) + assert ( + response.status_code == 400 + ), "Gets picked up by the duplicate privacy notice check" From 5ebf7a5f209030555712663a47507358036ade08 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 11 Jul 2023 15:44:10 -0500 Subject: [PATCH 4/9] Add the ability to pass in the record where the notice was served when saving privacy preferences. --- .../endpoints/privacy_preference_endpoints.py | 50 ++++++++++- src/fides/api/schemas/privacy_preference.py | 1 + tests/fixtures/application_fixtures.py | 18 ++++ .../test_privacy_preference_endpoints.py | 85 ++++++++++++++++++- 4 files changed, 151 insertions(+), 3 deletions(-) diff --git a/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py index 1fc5d9516d..141695bc96 100644 --- a/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py @@ -10,7 +10,12 @@ from loguru import logger from sqlalchemy import literal from sqlalchemy.orm import Query, Session -from starlette.status import HTTP_200_OK, HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND +from starlette.status import ( + HTTP_200_OK, + HTTP_400_BAD_REQUEST, + HTTP_404_NOT_FOUND, + HTTP_422_UNPROCESSABLE_ENTITY, +) from fides.api.api.deps import get_db from fides.api.api.v1.endpoints.consent_request_endpoints import ( @@ -81,6 +86,25 @@ router = APIRouter(tags=["Privacy Preference"], prefix=V1_URL_PREFIX) +def get_served_notice_history_id( + db: Session, served_notice_history_id: str +) -> ServedNoticeHistory: + """ + Helper method to load a ServedNoticeHistory record or throw a 404 + """ + logger.info("Finding ServedNoticeHistory with id '{}'", served_notice_history_id) + served_notice_history = ServedNoticeHistory.get( + db=db, object_id=served_notice_history_id + ) + if not served_notice_history: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No ServedNoticeHistory record found for id {served_notice_history_id}.", + ) + + return served_notice_history + + @router.post( CONSENT_REQUEST_PRIVACY_PREFERENCES_VERIFY, status_code=HTTP_200_OK, @@ -158,6 +182,25 @@ def verify_privacy_notice_and_historical_records( ) +def verify_valid_service_notice_history_records( + db: Session, data: PrivacyPreferencesRequest +): + """Verify service notice history records specified in the request are valid before saving privacy preferences""" + for preference in data.preferences: + if preference.served_notice_history_id: + served_notice_history = get_served_notice_history_id( + db, preference.served_notice_history_id + ) + if ( + served_notice_history.privacy_notice_history_id + != preference.privacy_notice_history_id + ): + raise HTTPException( + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"The ServedNoticeHistory record '{served_notice_history.id}' did not serve the PrivacyNoticeHistory record '{preference.privacy_notice_history_id}'.", + ) + + def extract_identity_from_provided_identity( identity: Optional[ProvidedIdentity], identity_type: ProvidedIdentityType ) -> Tuple[Optional[str], Optional[str]]: @@ -289,6 +332,8 @@ def save_privacy_preferences_with_verified_identity( for consent_option in data.preferences ], ) + verify_valid_service_notice_history_records(db, data) + consent_request, provided_identity = _get_consent_request_and_provided_identity( db=db, consent_request_id=consent_request_id, @@ -372,6 +417,7 @@ def _save_privacy_preferences_for_identities( if verified_provided_identity else None, "request_origin": request_data.request_origin, + "served_notice_history_id": privacy_preference.served_notice_history_id, "user_agent": request_data.user_agent, "user_geography": request_data.user_geography, "url_recorded": request_data.url_recorded, @@ -518,6 +564,8 @@ def save_privacy_preferences( ], ) + verify_valid_service_notice_history_records(db, data) + fides_user_provided_identity = get_or_create_fides_user_device_id_provided_identity( db=db, identity_data=data.browser_identity ) diff --git a/src/fides/api/schemas/privacy_preference.py b/src/fides/api/schemas/privacy_preference.py index 7297986c1f..23b1a743c0 100644 --- a/src/fides/api/schemas/privacy_preference.py +++ b/src/fides/api/schemas/privacy_preference.py @@ -23,6 +23,7 @@ class ConsentOptionCreate(FidesSchema): privacy_notice_history_id: str preference: UserConsentPreference + served_notice_history_id: Optional[str] class PrivacyPreferencesRequest(FidesSchema): diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 5907a1b30c..123bc52f71 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -1465,6 +1465,24 @@ def privacy_notice(db: Session) -> Generator: yield privacy_notice +@pytest.fixture(scope="function") +def served_notice_history( + db: Session, privacy_notice, fides_user_provided_identity +) -> Generator: + pref_1 = ServedNoticeHistory.create( + db=db, + data={ + "acknowledge_mode": False, + "serving_component": "overlay", + "fides_user_device_provided_identity_id": fides_user_provided_identity.id, + "privacy_notice_history_id": privacy_notice.privacy_notice_history_id, + }, + check_name=False, + ) + yield pref_1 + pref_1.delete(db) + + @pytest.fixture(scope="function") def privacy_notice_us_ca_provide(db: Session) -> Generator: privacy_notice = PrivacyNotice.create( diff --git a/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py index ae7e2c1854..2b226ead82 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py @@ -47,7 +47,9 @@ def verification_code(self) -> str: return "abcd" @pytest.fixture(scope="function") - def request_body(self, privacy_notice, verification_code, consent_policy): + def request_body( + self, privacy_notice, verification_code, consent_policy, served_notice_history + ): return { "browser_identity": {"ga_client_id": "test"}, "code": verification_code, @@ -55,6 +57,7 @@ def request_body(self, privacy_notice, verification_code, consent_policy): { "privacy_notice_history_id": privacy_notice.histories[0].id, "preference": "opt_out", + "served_notice_history_id": served_notice_history.id, } ], "policy_key": consent_policy.key, @@ -557,6 +560,35 @@ def test_set_duplicate_preferences_for_the_same_notice_in_one_request( response.status_code == 400 ), "Gets picked up by the duplicate privacy notice check" + @pytest.mark.usefixtures( + "subject_identity_verification_required", + ) + def test_save_preferences_invalid_served_notice_history_id( + self, + provided_identity_and_consent_request, + api_client, + verification_code, + request_body, + privacy_notice, + served_notice_history_us_ca_provide_for_fides_user, + ): + _, consent_request = provided_identity_and_consent_request + consent_request.cache_identity_verification_code(verification_code) + + request_body["preferences"][0][ + "served_notice_history_id" + ] = served_notice_history_us_ca_provide_for_fides_user.id + + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_PRIVACY_PREFERENCES_WITH_ID.format(consent_request_id=consent_request.id)}", + json=request_body, + ) + assert response.status_code == 422 + assert ( + response.json()["detail"] + == f"The ServedNoticeHistory record '{served_notice_history_us_ca_provide_for_fides_user.id}' did not serve the PrivacyNoticeHistory record '{privacy_notice.histories[0].id}'." + ) + @pytest.mark.usefixtures( "subject_identity_verification_required", "automatically_approved", @@ -578,6 +610,7 @@ def test_set_privacy_preferences( request_body, privacy_notice, privacy_notice_us_ca_provide, + served_notice_history, ): provided_identity, consent_request = provided_identity_and_consent_request consent_request.cache_identity_verification_code(verification_code) @@ -636,6 +669,14 @@ def test_set_privacy_preferences( db.refresh(first_privacy_preference_history_created) db.refresh(second_privacy_preference_history_created) + assert ( + first_privacy_preference_history_created.served_notice_history_id + == served_notice_history.id + ) + assert ( + second_privacy_preference_history_created.served_notice_history_id is None + ) + assert first_privacy_preference_history_created.privacy_request_id is not None assert second_privacy_preference_history_created.privacy_request_id is not None @@ -1064,7 +1105,13 @@ def url(self) -> str: return V1_URL_PREFIX + PRIVACY_PREFERENCES @pytest.fixture(scope="function") - def request_body(self, privacy_notice, consent_policy, privacy_experience_overlay): + def request_body( + self, + privacy_notice, + consent_policy, + privacy_experience_overlay, + served_notice_history, + ): return { "browser_identity": { "ga_client_id": "test", @@ -1074,6 +1121,7 @@ def request_body(self, privacy_notice, consent_policy, privacy_experience_overla { "privacy_notice_history_id": privacy_notice.histories[0].id, "preference": "opt_out", + "served_notice_history_id": served_notice_history.id, } ], "policy_key": consent_policy.key, @@ -1120,6 +1168,34 @@ def test_save_privacy_preferences_with_bad_notice( ) assert response.status_code == 400 + @pytest.mark.usefixtures( + "privacy_notice", + ) + def test_save_privacy_preferences_with_invalid_served_notice_history( + self, + api_client, + url, + request_body, + served_notice_history_us_ca_provide_for_fides_user, + ): + request_body["preferences"][0][ + "served_notice_history_id" + ] = served_notice_history_us_ca_provide_for_fides_user.id + response = api_client.patch(url, json=request_body) + assert response.status_code == 422 + + @pytest.mark.usefixtures( + "privacy_notice", + ) + def test_save_privacy_preferences_with_served_notice_history_not_found( + self, api_client, url, request_body + ): + request_body["preferences"][0][ + "served_notice_history_id" + ] = "bad_served_notice_history_id" + response = api_client.patch(url, json=request_body) + assert response.status_code == 404 + def test_save_privacy_preferences_bad_experience_id( self, api_client, @@ -1150,6 +1226,7 @@ def test_save_privacy_preferences_with_respect_to_fides_user_device_id( request_body, privacy_notice, privacy_experience_overlay, + served_notice_history, ): """Assert CurrentPrivacyPreference records were updated and PrivacyPreferenceHistory records were created for recordkeeping with respect to the fides user device id in the request @@ -1221,6 +1298,10 @@ def test_save_privacy_preferences_with_respect_to_fides_user_device_id( assert privacy_preference_history.anonymized_ip_address == masked_ip assert privacy_preference_history.url_recorded is None assert privacy_preference_history.method == ConsentMethod.button + assert ( + privacy_preference_history.served_notice_history_id + == served_notice_history.id + ) # Privacy request created and queued because a privacy notice has system wide enforcement assert privacy_preference_history.privacy_request_id is not None From 12b61dafaeda8b2d72fb39a46279ec1cc578212a Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 11 Jul 2023 16:02:09 -0500 Subject: [PATCH 5/9] Surface served_notice_history_id in the consent reporting endpoint. --- src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py | 3 +++ src/fides/api/schemas/privacy_preference.py | 3 +++ tests/fixtures/application_fixtures.py | 2 ++ .../ops/api/v1/endpoints/test_privacy_preference_endpoints.py | 2 ++ 4 files changed, 10 insertions(+) diff --git a/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py index 141695bc96..f2ac582595 100644 --- a/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py @@ -670,6 +670,9 @@ def get_historical_consent_report( "truncated_ip_address" ), PrivacyPreferenceHistory.method.label("method"), + PrivacyPreferenceHistory.served_notice_history_id.label( + "served_notice_history_id" + ), ) .outerjoin( PrivacyRequest, diff --git a/src/fides/api/schemas/privacy_preference.py b/src/fides/api/schemas/privacy_preference.py index 23b1a743c0..bea95db0fe 100644 --- a/src/fides/api/schemas/privacy_preference.py +++ b/src/fides/api/schemas/privacy_preference.py @@ -139,6 +139,9 @@ class ConsentReportingSchema(FidesSchema): ) truncated_ip_address: Optional[str] = Field(title="Truncated ip address") method: Optional[ConsentMethod] = Field(title="Method of consent preference") + served_notice_history_id: Optional[str] = Field( + title="The id of the record where the notice was served to the end user" + ) class CurrentPrivacyPreferenceSchema(FidesSchema): diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 123bc52f71..94eece4b5b 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -2159,6 +2159,7 @@ def privacy_preference_history( provided_identity_and_consent_request, privacy_notice, privacy_experience_privacy_center, + served_notice_history, ): provided_identity, consent_request = provided_identity_and_consent_request privacy_notice_history = privacy_notice.histories[0] @@ -2178,6 +2179,7 @@ def privacy_preference_history( "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/324.42 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/425.24", "user_geography": "us_ca", "url_recorded": "example.com/privacy_center", + "served_notice_history_id": served_notice_history.id, }, check_name=False, ) diff --git a/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py index 2b226ead82..ec86013dff 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py @@ -1424,6 +1424,7 @@ def test_get_historical_preferences( generate_auth_header, privacy_preference_history, privacy_request_with_consent_policy, + served_notice_history, system, privacy_experience_privacy_center, ) -> None: @@ -1495,6 +1496,7 @@ def test_get_historical_preferences( response_body["privacy_experience_id"] == privacy_experience_privacy_center.id ) + assert response_body["served_notice_history_id"] == served_notice_history.id def test_get_historical_preferences_user_geography_unsupported( self, From 3298ee694688acbdea6125a87a62c84a6ec0b113 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 11 Jul 2023 16:58:39 -0500 Subject: [PATCH 6/9] Address mypy failures. --- .../endpoints/privacy_preference_endpoints.py | 52 ++++++++++++------- src/fides/api/models/privacy_notice.py | 4 +- src/fides/api/models/privacy_preference.py | 21 +++++--- src/fides/api/util/consent_util.py | 4 +- 4 files changed, 51 insertions(+), 30 deletions(-) diff --git a/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py index f2ac582595..757eb68c44 100644 --- a/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py @@ -24,6 +24,7 @@ from fides.api.api.v1.endpoints.privacy_request_endpoints import ( create_privacy_request_func, ) +from fides.api.custom_types import SafeStr from fides.api.db.seed import DEFAULT_CONSENT_POLICY from fides.api.models.fides_user import FidesUser from fides.api.models.privacy_experience import PrivacyExperience @@ -152,7 +153,7 @@ def consent_request_verify_for_privacy_preferences( def verify_privacy_notice_and_historical_records( - db: Session, notice_history_list: List[str] + db: Session, notice_history_list: List[SafeStr] ) -> None: """ Used when saving privacy preferences: runs a check that makes sure all the privacy notice histories referenced by @@ -184,7 +185,7 @@ def verify_privacy_notice_and_historical_records( def verify_valid_service_notice_history_records( db: Session, data: PrivacyPreferencesRequest -): +) -> None: """Verify service notice history records specified in the request are valid before saving privacy preferences""" for preference in data.preferences: if preference.served_notice_history_id: @@ -242,7 +243,7 @@ def anonymize_ip_address(ip_address: Optional[str]) -> Optional[str]: def _get_request_origin_and_config( - db: Session, data: PrivacyPreferencesRequest + db: Session, data: Union[PrivacyPreferencesRequest, NoticesServedRequest] ) -> Tuple[Optional[str], Optional[str]]: """ Extract details to save with the privacy preferences preferences from the @@ -341,7 +342,7 @@ def save_privacy_preferences_with_verified_identity( ) ( - provided_identity, + provided_identity_verified, fides_user_provided_identity, ) = classify_identities_for_privacy_center_consent_reporting( db=db, @@ -350,14 +351,18 @@ def save_privacy_preferences_with_verified_identity( ) logger.info("Saving privacy preferences") + + request_data = supplement_request_with_user_and_experience_details( + db, request, data, resource_type=PrivacyPreferencesCreate + ) + assert isinstance(request_data, PrivacyPreferencesCreate) # For mypy + return _save_privacy_preferences_for_identities( db=db, consent_request=consent_request, - verified_provided_identity=provided_identity, + verified_provided_identity=provided_identity_verified, fides_user_provided_identity=fides_user_provided_identity, - request_data=supplement_request_with_user_and_experience_details( - db, request, data, resource_type=PrivacyPreferencesCreate - ), + request_data=request_data, ) @@ -571,14 +576,17 @@ def save_privacy_preferences( ) logger.info("Saving privacy preferences with respect to fides user device id") + request_data = supplement_request_with_user_and_experience_details( + db, request, data, resource_type=PrivacyPreferencesCreate + ) + + assert isinstance(request_data, PrivacyPreferencesCreate) # For mypy return _save_privacy_preferences_for_identities( db=db, consent_request=None, verified_provided_identity=None, fides_user_provided_identity=fides_user_provided_identity, - request_data=supplement_request_with_user_and_experience_details( - db, request, data, resource_type=PrivacyPreferencesCreate - ), + request_data=request_data, ) @@ -718,20 +726,22 @@ def save_notices_served( ) logger.info("Recording notices served with respect to fides user device id") + request_data = supplement_request_with_user_and_experience_details( + db, request, data, resource_type=NoticesServedCreate + ) + assert isinstance(request_data, NoticesServedCreate) # For mypy return _save_notices_served_for_identities( db=db, verified_provided_identity=None, fides_user_provided_identity=fides_user_provided_identity, - request_data=supplement_request_with_user_and_experience_details( - db, request, data, resource_type=NoticesServedCreate - ), + request_data=request_data, ) def classify_identities_for_privacy_center_consent_reporting( db: Session, provided_identity: ProvidedIdentity, - browser_identity: Optional[Identity], + browser_identity: Identity, ) -> Tuple[Optional[ProvidedIdentity], Optional[ProvidedIdentity]]: """For consent reporting purposes, we separate out the type of identity that identifies the user @@ -793,7 +803,7 @@ def save_notices_served_via_privacy_center( ) ( - provided_identity, + provided_identity_verified, fides_user_provided_identity, ) = classify_identities_for_privacy_center_consent_reporting( db=db, @@ -802,11 +812,13 @@ def save_notices_served_via_privacy_center( ) logger.info("Saving notices served for privacy center") + request_data = supplement_request_with_user_and_experience_details( + db, request, data, resource_type=NoticesServedCreate + ) + assert isinstance(request_data, NoticesServedCreate) return _save_notices_served_for_identities( db=db, - verified_provided_identity=provided_identity, + verified_provided_identity=provided_identity_verified, fides_user_provided_identity=fides_user_provided_identity, - request_data=supplement_request_with_user_and_experience_details( - db, request, data, resource_type=NoticesServedCreate - ), + request_data=request_data, ) diff --git a/src/fides/api/models/privacy_notice.py b/src/fides/api/models/privacy_notice.py index 4948668d89..1d999b4d69 100644 --- a/src/fides/api/models/privacy_notice.py +++ b/src/fides/api/models/privacy_notice.py @@ -164,8 +164,8 @@ class PrivacyNoticeBase: # Attribute that can be temporarily cached as the result of "get_related_privacy_notices" # for a given user, for surfacing CurrentPrivacyPreferences for the user. - current_preference: Optional[UserConsentPreference] = None - outdated_preference: Optional[UserConsentPreference] = None + current_preference: Optional[str] = None + outdated_preference: Optional[str] = None # Attributes that can be temporarily cached on the notice to see if the most # recent version or a previous version of a notice have ever been served to the user current_served: Optional[bool] = None diff --git a/src/fides/api/models/privacy_preference.py b/src/fides/api/models/privacy_preference.py index a868e93f86..f8dc734d2f 100644 --- a/src/fides/api/models/privacy_preference.py +++ b/src/fides/api/models/privacy_preference.py @@ -258,13 +258,16 @@ def save_notice_served_and_last_notice_served( "fides_user_device_provided_identity_id": created_served_notice_history.fides_user_device_provided_identity_id, } - upserted_last_served_notice_record: LastServedNotice = upsert_last_saved_record( + upserted_last_served_notice_record = upsert_last_saved_record( db, created_historical_record=created_served_notice_history, current_record_class=LastServedNotice, privacy_notice_history=privacy_notice_history, current_record_data=last_served_data, ) + assert isinstance( + upserted_last_served_notice_record, LastServedNotice + ) # For mypy return created_served_notice_history, upserted_last_served_notice_record @@ -395,13 +398,15 @@ def create_history_and_upsert_current_preference( "fides_user_device_provided_identity_id": created_privacy_preference_history.fides_user_device_provided_identity_id, } - current_preference: CurrentPrivacyPreference = upsert_last_saved_record( + current_preference = upsert_last_saved_record( db, created_historical_record=created_privacy_preference_history, current_record_class=CurrentPrivacyPreference, privacy_notice_history=privacy_notice_history, current_record_data=current_privacy_preference_data, ) + assert isinstance(current_preference, CurrentPrivacyPreference) # For mypy + return created_privacy_preference_history, current_preference @@ -583,11 +588,13 @@ def upsert_last_saved_record( If we have historical records for multiple identities, and we determine they are the same identity, their preferences are consolidated into the same record. """ - existing_record_for_provided_identity = None + existing_record_for_provided_identity: Optional[ + Union[CurrentPrivacyPreference, LastServedNotice] + ] = None # Check if we have "current" records for the ProvidedIdentity (usu an email or phone)/Privacy Notice if created_historical_record.provided_identity_id: existing_record_for_provided_identity = ( - db.query(current_record_class) + db.query(current_record_class) # type: ignore[assignment] .filter( current_record_class.provided_identity_id == created_historical_record.provided_identity_id, @@ -598,10 +605,12 @@ def upsert_last_saved_record( ) # Check if we have "current" records for the Fides User Device ID and the Privacy Notice - existing_record_for_fides_user_device = None + existing_record_for_fides_user_device: Optional[ + Union[CurrentPrivacyPreference, LastServedNotice] + ] = None if created_historical_record.fides_user_device_provided_identity_id: existing_record_for_fides_user_device = ( - db.query(current_record_class) + db.query(current_record_class) # type: ignore[assignment] .filter( current_record_class.fides_user_device_provided_identity_id == created_historical_record.fides_user_device_provided_identity_id, diff --git a/src/fides/api/util/consent_util.py b/src/fides/api/util/consent_util.py index 7463b8e0f3..8ee7c8d4bb 100644 --- a/src/fides/api/util/consent_util.py +++ b/src/fides/api/util/consent_util.py @@ -235,12 +235,12 @@ def get_fides_user_device_id_provided_identity( def get_or_create_fides_user_device_id_provided_identity( db: Session, - identity_data: Identity, + identity_data: Optional[Identity], ) -> ProvidedIdentity: """Gets an existing fides user device id provided identity or creates one if it doesn't exist. Raises an error if no fides user device id is supplied. """ - if not identity_data.fides_user_device_id: + if not identity_data or not identity_data.fides_user_device_id: raise HTTPException( HTTP_422_UNPROCESSABLE_ENTITY, detail="Fides user device id not found in identity data", From 36b1c7aa993cd3b2d9d4de16d78e051d3715cfa7 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 11 Jul 2023 17:04:47 -0500 Subject: [PATCH 7/9] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a8a2127df..81ebe3dd63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,9 @@ The types of changes are: - Bumped supported Python versions to `3.10.12`, `3.9.17`, and `3.8.17` [#3733](https://github.com/ethyca/fides/pull/3733) - Add polyfill service to fides-js route [#3759](https://github.com/ethyca/fides/pull/3759) +### Added +- Record when consent is served [#3777](https://github.com/ethyca/fides/pull/3777) + ## [2.16.0](https://github.com/ethyca/fides/compare/2.15.1...2.16.0) ### Added From c1f78bebded35f7e658ae0085edc27f526f66c66 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 12 Jul 2023 13:30:58 -0500 Subject: [PATCH 8/9] Add dataset annotations for new tables/columns. --- .fides/db_dataset.yml | 130 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index a73b4c3261..c18d939d08 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -2204,6 +2204,10 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: served_notice_history_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: userregistration description: 'Records the registration status of this Fides deployment' data_categories: null @@ -2277,4 +2281,130 @@ dataset: - name: updated_at data_categories: - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: lastservednotice + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + fields: + - name: created_at + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: fides_user_device_provided_identity_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_notice_history_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_notice_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: provided_identity_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: served_notice_history_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: updated_at + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: servednoticehistory + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + fields: + - name: acknowledge_mode + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: anonymized_ip_address + data_categories: + - user + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: created_at + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: updated_at + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: email + data_categories: + - user + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: fides_user_device + data_categories: + - user + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: fides_user_device_provided_identity_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: hashed_email + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: hashed_fides_user_device + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: hashed_phone_number + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: phone_number + data_categories: + - user + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_experience_config_history_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_experience_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_notice_history_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: provided_identity_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: request_origin + data_categories: + - user + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: serving_component + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: updated_at + description: 'Fides Generated Description for Column: updated_at' + data_categories: [ ] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: url_recorded + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: user_agent + data_categories: + - user + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: user_geography + data_categories: + - user data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified \ No newline at end of file From 6ce9488072de941d41240a17ae143a799fcd5b3b Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 13 Jul 2023 12:49:39 -0500 Subject: [PATCH 9/9] Fix some typos and expand docstrings. --- .../endpoints/privacy_preference_endpoints.py | 19 ++++++++++++------- src/fides/api/models/privacy_experience.py | 8 ++++---- src/fides/api/models/privacy_preference.py | 2 +- src/fides/api/schemas/privacy_preference.py | 2 +- .../test_privacy_experience_endpoints.py | 3 ++- 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py index 757eb68c44..0430c17946 100644 --- a/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py @@ -87,7 +87,7 @@ router = APIRouter(tags=["Privacy Preference"], prefix=V1_URL_PREFIX) -def get_served_notice_history_id( +def get_served_notice_history( db: Session, served_notice_history_id: str ) -> ServedNoticeHistory: """ @@ -186,10 +186,12 @@ def verify_privacy_notice_and_historical_records( def verify_valid_service_notice_history_records( db: Session, data: PrivacyPreferencesRequest ) -> None: - """Verify service notice history records specified in the request are valid before saving privacy preferences""" + """Verify service notice history records specified in the request (that link an event that a notice was + served to the event that saved the preference) are valid before saving privacy preferences + """ for preference in data.preferences: if preference.served_notice_history_id: - served_notice_history = get_served_notice_history_id( + served_notice_history = get_served_notice_history( db, preference.served_notice_history_id ) if ( @@ -246,8 +248,8 @@ def _get_request_origin_and_config( db: Session, data: Union[PrivacyPreferencesRequest, NoticesServedRequest] ) -> Tuple[Optional[str], Optional[str]]: """ - Extract details to save with the privacy preferences preferences from the - Experience history if applicable: request origin (privacy center, overlay) + Extract details to save with the privacy preferences or notices + served from the Experience history if applicable: request origin (privacy center, overlay) and the Experience Config history. Additionally validate that the experience config history is valid if supplied. @@ -486,7 +488,9 @@ def _save_notices_served_for_identities( request_data: NoticesServedCreate, ) -> List[LastServedNotice]: """ - Common code to that saves that notices have been served (both historical and current records) + Common code that saves that notices have been served (both historical and current records). + We store a historical record for every single time a notice was served to the user in the frontend, + and a separate "last served notice" for just the last time a notice was served to a given user. """ created_notices_served: List[ServedNoticeHistory] = [] upserted_last_served: List[LastServedNotice] = [] @@ -743,7 +747,8 @@ def classify_identities_for_privacy_center_consent_reporting( provided_identity: ProvidedIdentity, browser_identity: Identity, ) -> Tuple[Optional[ProvidedIdentity], Optional[ProvidedIdentity]]: - """For consent reporting purposes, we separate out the type of identity that identifies the user + """For consent reporting purposes, we distinguish which type of identities we have that + identity the user. We want to classify the "provided_identity" as an identifier saved against an email or phone, and the "fides_user_provided_identity" as an identifier saved against the fides user device id. diff --git a/src/fides/api/models/privacy_experience.py b/src/fides/api/models/privacy_experience.py index be46e78382..d782e23e94 100644 --- a/src/fides/api/models/privacy_experience.py +++ b/src/fides/api/models/privacy_experience.py @@ -542,7 +542,7 @@ def upsert_privacy_experiences_after_config_update( def cache_saved_preference_on_notice( db: Session, notice: PrivacyNotice, fides_user_provided_identity: ProvidedIdentity ) -> None: - """At runtime, cache any saved preference values on the privacy notice""" + """At runtime, cache any previously saved preference values for the given user on the privacy notice""" saved_preference: Optional[ CurrentPrivacyPreference ] = CurrentPrivacyPreference.get_preference_for_notice_and_fides_user_device( @@ -563,8 +563,8 @@ def cache_saved_preference_on_notice( def cache_notice_served( db: Session, notice: PrivacyNotice, fides_user_provided_identity: ProvidedIdentity ) -> None: - """At runtime, cache if the current notice was served to the user, or a previous version - of the notice, if applicable""" + """At runtime, cache if the current notice or a previous version of the notice was served to the user + if applicable""" served_notice: Optional[ LastServedNotice ] = LastServedNotice.get_last_served_for_notice_and_fides_user_device( @@ -573,7 +573,7 @@ def cache_notice_served( privacy_notice=notice, ) if served_notice: - # Temporarily cache the preference for the given fides user device id in memory. + # Temporarily cache that the notice was served for the given fides user device id in memory. if served_notice.served_latest_version: notice.current_served = True notice.outdated_served = None diff --git a/src/fides/api/models/privacy_preference.py b/src/fides/api/models/privacy_preference.py index f8dc734d2f..2da3838220 100644 --- a/src/fides/api/models/privacy_preference.py +++ b/src/fides/api/models/privacy_preference.py @@ -288,7 +288,7 @@ class PrivacyPreferenceHistory(ConsentReportingMixin, Base): # Whether the user wants to opt in, opt out, or has acknowledged the notice preference = Column(EnumColumn(UserConsentPreference), nullable=False, index=True) - # The privacy request created to propage the preferences + # The privacy request created to propagate the preferences privacy_request_id = Column( String, ForeignKey(PrivacyRequest.id, ondelete="SET NULL"), index=True ) diff --git a/src/fides/api/schemas/privacy_preference.py b/src/fides/api/schemas/privacy_preference.py index bea95db0fe..4db0ef1e1f 100644 --- a/src/fides/api/schemas/privacy_preference.py +++ b/src/fides/api/schemas/privacy_preference.py @@ -69,7 +69,7 @@ class NoticesServedRequest(FidesSchema): class NoticesServedCreate(NoticesServedRequest): - """Schema used on the backend only where we supplement the NoticesServedRequest requets body + """Schema used on the backend only where we supplement the NoticesServedRequest request body with information obtained from the request headers and the experience""" anonymized_ip_address: Optional[str] diff --git a/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py index d04d8e4417..a5a851a68b 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py @@ -591,10 +591,11 @@ def test_get_privacy_experiences_fides_user_device_id_filter( ) assert resp.status_code == 200 data = resp.json()["items"][0] - # Assert current preference is displayed for fides user device id + # Assert outdated preference is displayed for fides user device id assert data["privacy_notices"][0]["consent_mechanism"] == "opt_in" assert data["privacy_notices"][0]["default_preference"] == "opt_out" assert data["privacy_notices"][0]["current_preference"] is None assert data["privacy_notices"][0]["outdated_preference"] == "opt_in" + # Assert outdated served is displayed for fides user device id assert data["privacy_notices"][0]["current_served"] is None assert data["privacy_notices"][0]["outdated_served"] is True