[WIP] Add Remaining TCF Overlay Contents (#3966)

requirements.txt

@@ -13,7 +13,7 @@ expandvars==0.9.0
 fastapi[all]==0.89.1
 fastapi-caching[redis]==0.3.0
 fastapi-pagination[sqlalchemy]~= 0.10.0
-fideslang @ git+https://github.com/ethyca/fideslang.git@cf6675e01269d96e4f244c215e2a66aa654e4fae#egg=fideslang
+fideslang @ git+https://github.com/ethyca/fideslang.git@6694dcc2c653172d95db6665fc2a3da62b809489#egg=fideslang
 fideslog==1.2.10
 firebase-admin==5.3.0
 GitPython==3.1.31

src/fides/api/alembic/migrations/versions/3ceb7162348a_store_preferences_against_system_fides_key.py

@@ -0,0 +1,117 @@
+"""store preferences against system id
+
+Revision ID: 3ceb7162348a
+Revises: 66df7d9b8103
+Create Date: 2023-08-20 22:54:24.855291
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "3ceb7162348a"
+down_revision = "66df7d9b8103"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "currentprivacypreference",
+        sa.Column("system_fides_key", sa.String(), nullable=True),
+    )
+    op.create_unique_constraint(
+        "fides_user_device_identity_system_fides_key",
+        "currentprivacypreference",
+        ["fides_user_device_provided_identity_id", "system_fides_key"],
+    )
+    op.create_unique_constraint(
+        "identity_system_fides_key",
+        "currentprivacypreference",
+        ["provided_identity_id", "system_fides_key"],
+    )
+    op.create_index(
+        op.f("ix_currentprivacypreference_system_fides_key"),
+        "currentprivacypreference",
+        ["system_fides_key"],
+        unique=False,
+    )
+
+    op.add_column(
+        "lastservednotice", sa.Column("system_fides_key", sa.String(), nullable=True)
+    )
+    op.create_index(
+        op.f("ix_lastservednotice_system_fides_key"),
+        "lastservednotice",
+        ["system_fides_key"],
+        unique=False,
+    )
+    op.create_unique_constraint(
+        "last_served_fides_user_device_identity_system_fides_key",
+        "lastservednotice",
+        ["fides_user_device_provided_identity_id", "system_fides_key"],
+    )
+    op.create_unique_constraint(
+        "last_served_identity_system_fides_key",
+        "lastservednotice",
+        ["provided_identity_id", "system_fides_key"],
+    )
+
+    op.add_column(
+        "privacypreferencehistory",
+        sa.Column("system_fides_key", sa.String(), nullable=True),
+    )
+    op.create_index(
+        op.f("ix_privacypreferencehistory_system_fides_key"),
+        "privacypreferencehistory",
+        ["system_fides_key"],
+        unique=False,
+    )
+
+    op.add_column(
+        "servednoticehistory", sa.Column("system_fides_key", sa.String(), nullable=True)
+    )
+    op.create_index(
+        op.f("ix_servednoticehistory_system_fides_key"),
+        "servednoticehistory",
+        ["system_fides_key"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        op.f("ix_servednoticehistory_system_fides_key"),
+        table_name="servednoticehistory",
+    )
+    op.drop_column("servednoticehistory", "system_fides_key")
+    op.drop_index(
+        op.f("ix_privacypreferencehistory_system_fides_key"),
+        table_name="privacypreferencehistory",
+    )
+    op.drop_column("privacypreferencehistory", "system_fides_key")
+    op.drop_constraint(
+        "last_served_identity_system_fides_key", "lastservednotice", type_="unique"
+    )
+    op.drop_constraint(
+        "last_served_fides_user_device_identity_system_fides_key",
+        "lastservednotice",
+        type_="unique",
+    )
+    op.drop_index(
+        op.f("ix_lastservednotice_system_fides_key"), table_name="lastservednotice"
+    )
+    op.drop_column("lastservednotice", "system_fides_key")
+    op.drop_index(
+        op.f("ix_currentprivacypreference_system_fides_key"),
+        table_name="currentprivacypreference",
+    )
+    op.drop_constraint(
+        "identity_system_fides_key", "currentprivacypreference", type_="unique"
+    )
+    op.drop_constraint(
+        "fides_user_device_identity_system_fides_key",
+        "currentprivacypreference",
+        type_="unique",
+    )
+    op.drop_column("currentprivacypreference", "system_fides_key")

src/fides/api/api/v1/endpoints/consent_settings_endpoints.py

@@ -19,11 +19,10 @@
 
 @router.get(
     urls.CONSENT_SETTINGS,
-    dependencies=[Security(verify_oauth_client, scopes=[scopes.CONSENT_SETTINGS_READ])],
     response_model=ConsentSettingsResponseSchema,
 )
 def get_consent_settings(*, db: Session = Depends(deps.get_db)) -> ConsentSettings:
-    """Returns organization-wide consent settings."""
+    """Public endpoint that returns organization-wide consent settings."""
     logger.info("Getting organization-wide consent settings")
     return ConsentSettings.get_or_create_with_defaults(db)
 

src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py

@@ -135,10 +135,21 @@ def privacy_experience_list(
     'show_disabled' query params are passed along to further filter
     notices as well.
 
-    'fides_user_device_id' query param will stash the current preferences of the given user
-    alongside each notice where applicable.
+    :param db:
+    :param params:
+    :param show_disabled: If False, returns enabled Experiences and Notices
+    :param region: Return the Experiences for the given region
+    :param component: Returns Experiences of the given component type
+    :param has_notices: Return if the Experience has content
+    :param has_config: Return if the Experience has copy
+    :param fides_user_device_id: Supplement the response with current saved preferences of the given user
+    :param systems_applicable: Only return embedded Notices associated with systems.
+    :param request:
+    :param response:
+    :return:
     """
     logger.info("Finding all Privacy Experiences with pagination params '{}'", params)
+    content_required: bool = has_notices  # Renaming confusing query param
     fides_user_provided_identity: Optional[ProvidedIdentity] = None
     if fides_user_device_id:
         try:
@@ -213,7 +224,8 @@ def privacy_experience_list(
                 fields=PRIVACY_EXPERIENCE_ESCAPE_FIELDS,
             )
 
-        if not (has_notices and not content_exists):
+        if not (content_required and not content_exists):
+            # Append experience unless we're missing required content
             results.append(privacy_experience)
 
     return fastapi_paginate(results, params=params)
@@ -230,7 +242,7 @@ def embed_experience_details(
     """
     At runtime embeds relevant Experience contents where applicable:
         - Privacy Notices
-        - TCF Components: purposes, special purposes, vendors, features, and special features
+        - TCF Components: purposes, special purposes, vendors, systems, features, and special features
 
     Returns whether content exists on the experience.
     """
@@ -239,14 +251,18 @@ def embed_experience_details(
     for component in TCF_COMPONENT_MAPPING:
         setattr(privacy_experience, component, [])
 
-    # Temporarily add TCF Contents to the Experience object if applicable
+    # Fetch the base TCF Contents
     tcf_contents: TCFExperienceContents = privacy_experience.get_related_tcf_contents(
         db, fides_user_provided_identity
     )
+    has_tcf_contents: bool = any(
+        getattr(tcf_contents, component) for component in TCF_COMPONENT_MAPPING
+    )
+    # Supplement the Privacy Experience with TCF Contents if applicable
     for component in TCF_COMPONENT_MAPPING:
         setattr(privacy_experience, component, getattr(tcf_contents, component))
 
-    # Temporarily add Privacy Notices to the Experience object if applicable
+    # Add Privacy Notices to the Experience if applicable
     privacy_notices: List[
         PrivacyNotice
     ] = privacy_experience.get_related_privacy_notices(
@@ -264,6 +280,4 @@ def embed_experience_details(
         ]
     privacy_experience.privacy_notices = privacy_notices
 
-    return bool(privacy_notices) or any(
-        getattr(tcf_contents, component) for component in TCF_COMPONENT_MAPPING
-    )
+    return bool(privacy_notices) or has_tcf_contents

src/fides/api/api/v1/endpoints/privacy_preference_endpoints.py

@@ -454,7 +454,8 @@ def save_tcf_preference(
         )
         return current_preference
 
-    # Save TCF preferences separately with respect to purpose, special purpose, vendor, feature, and/or special feature.
+    # Save TCF preferences individually with respect to purpose, special purpose, vendor,
+    # feature, special feature, and/or system fides key.
     # Currently, we don't attempt to propagate these preferences to third party systems.
     for tcf_preference_field, field_name in TCF_PREFERENCES_FIELD_MAPPING.items():
         for preference in getattr(request_data, tcf_preference_field):

src/fides/api/models/privacy_experience.py

@@ -222,6 +222,7 @@ class PrivacyExperience(Base):
     tcf_vendors: List = []
     tcf_features: List = []
     tcf_special_features: List = []
+    tcf_systems: List = []
 
     # Attribute that is cached on the PrivacyExperience object by "get_should_show_banner", calculated at runtime
     show_banner: bool