From 349d0ee5f940bd5df9d870c5dc5468fe110f9a0d Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 27 Nov 2023 10:38:21 -0600 Subject: [PATCH 01/15] Add TCFPublisherOverrides table and PrivacyDeclaration.purpose hybrid property. --- .../5225ea4de265_tcf_publisher_overrides.py | 51 +++++++++++++++++++ src/fides/api/db/base.py | 1 + src/fides/api/models/sql_models.py | 21 ++++++++ .../api/models/tcf_publisher_overrides.py | 13 +++++ .../api/util/tcf/tcf_experience_contents.py | 1 + tests/ctl/core/test_system.py | 45 ++++++++++++++++ .../api/v1/endpoints/test_config_endpoints.py | 4 +- 7 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py create mode 100644 src/fides/api/models/tcf_publisher_overrides.py diff --git a/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py b/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py new file mode 100644 index 0000000000..98ef26edf7 --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py @@ -0,0 +1,51 @@ +"""tcf_publisher_overrides + +Revision ID: 5225ea4de265 +Revises: 1af6950f4625 +Create Date: 2023-11-27 15:35:07.679747 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "5225ea4de265" +down_revision = "1af6950f4625" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "tcfpublisheroverrides", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("purpose", sa.Integer(), nullable=False), + sa.Column("is_included", sa.Boolean(), server_default="t", nullable=True), + sa.Column("required_legal_basis", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_tcfpublisheroverrides_id"), + "tcfpublisheroverrides", + ["id"], + unique=False, + ) + + +def downgrade(): + op.drop_index( + op.f("ix_tcfpublisheroverrides_id"), table_name="tcfpublisheroverrides" + ) + op.drop_table("tcfpublisheroverrides") diff --git a/src/fides/api/db/base.py b/src/fides/api/db/base.py index 92caa47ea2..b27d20c258 100644 --- a/src/fides/api/db/base.py +++ b/src/fides/api/db/base.py @@ -39,3 +39,4 @@ from fides.api.models.system_compass_sync import SystemCompassSync from fides.api.models.system_history import SystemHistory from fides.api.models.system_manager import SystemManager +from fides.api.models.tcf_publisher_overrides import TCFPublisherOverrides diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index 636039e9d5..a6fccc9e78 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -9,6 +9,8 @@ from enum import Enum as EnumType from typing import Any, Dict, List, Optional, Set, Type, TypeVar +from fideslang import MAPPED_PURPOSES_BY_DATA_USE +from fideslang.gvl import MAPPED_PURPOSES, MappedPurpose from fideslang.models import DataCategory as FideslangDataCategory from fideslang.models import Dataset as FideslangDataset from pydantic import BaseModel @@ -27,6 +29,7 @@ ) from sqlalchemy.dialects.postgresql import ARRAY, BIGINT, BYTEA from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import Session, relationship from sqlalchemy.sql import func from sqlalchemy.sql.sqltypes import DateTime @@ -514,6 +517,24 @@ def create( """Overrides base create to avoid unique check on `name` column""" return super().create(db=db, data=data, check_name=check_name) + @hybrid_property + def purpose(self) -> Optional[int]: + """Return the Purpose ID (not Special Purpose ID) if the data use maps to a Purpose + + If data use maps to a Special Purpose, None is returned here. This is for Purposes only. + """ + mapped_purpose: Optional[MappedPurpose] = MAPPED_PURPOSES_BY_DATA_USE.get( + self.data_use + ) + if not mapped_purpose: + return None + + return ( + mapped_purpose.id + if MAPPED_PURPOSES.get(mapped_purpose.id) == mapped_purpose + else None + ) + class SystemModel(BaseModel): fides_key: str diff --git a/src/fides/api/models/tcf_publisher_overrides.py b/src/fides/api/models/tcf_publisher_overrides.py new file mode 100644 index 0000000000..20e8d401a4 --- /dev/null +++ b/src/fides/api/models/tcf_publisher_overrides.py @@ -0,0 +1,13 @@ +from sqlalchemy import Boolean, Column, Integer, String + +from fides.api.db.base_class import Base + + +class TCFPublisherOverrides(Base): + """ + Stores TCF Publisher Overrides + """ + + purpose = Column(Integer, nullable=False) + is_included = Column(Boolean, server_default="t", default=True) + required_legal_basis = Column(String) diff --git a/src/fides/api/util/tcf/tcf_experience_contents.py b/src/fides/api/util/tcf/tcf_experience_contents.py index 88f94d88fb..c5213a5852 100644 --- a/src/fides/api/util/tcf/tcf_experience_contents.py +++ b/src/fides/api/util/tcf/tcf_experience_contents.py @@ -174,6 +174,7 @@ def get_matching_privacy_declarations(db: Session) -> Query: PrivacyDeclaration.legal_basis_for_processing, PrivacyDeclaration.features, PrivacyDeclaration.retention_period, + PrivacyDeclaration.flexible_legal_basis_for_processing, ) .outerjoin(PrivacyDeclaration, System.id == PrivacyDeclaration.system_id) .filter( diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index 04d40f6085..32902f937e 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -345,6 +345,51 @@ def test_scan_system_okta_fail( ) +class TestPrivacyDeclarationPurpose: + def test_privacy_declaration_purpose(self): + pd = PrivacyDeclaration( + name="declaration-name", + data_categories=[], + data_use="analytics.reporting.campaign_insights", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ingress=None, + egress=None, + ) + + assert pd.purpose == 9 + + def test_privacy_declaration_special_purpose(self): + """Special purposes are not returned under the purpose hybrid property""" + pd = PrivacyDeclaration( + name="declaration-name", + data_categories=[], + data_use="essential.service.security", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ingress=None, + egress=None, + ) + + assert pd.purpose is None + + def test_privacy_declaration_non_tcf_data_use(self): + pd = PrivacyDeclaration( + name="declaration-name", + data_categories=[], + data_use="essential", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ingress=None, + egress=None, + ) + + assert pd.purpose is None + + class TestUpsertCookies: @pytest.fixture() async def test_cookie_system( diff --git a/tests/ops/api/v1/endpoints/test_config_endpoints.py b/tests/ops/api/v1/endpoints/test_config_endpoints.py index a24c094d02..da54a35ed7 100644 --- a/tests/ops/api/v1/endpoints/test_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_config_endpoints.py @@ -970,7 +970,9 @@ def test_get_config( } for key in config.keys(): - assert (key in allowed_top_level_config_keys), "Unexpected config API change, please review with Ethyca security team" + assert ( + key in allowed_top_level_config_keys + ), "Unexpected config API change, please review with Ethyca security team" assert "security" in config assert "user" in config From b0637f31743e52768cb420bfc47fb57e2ab70e1d Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 28 Nov 2023 11:28:24 -0600 Subject: [PATCH 02/15] Take publisher overrides into account when querying privacy declarations that will form the basis of the TCF experience. - Define hybrid property PrivacyDeclaration.overridden_legal_basis_for_processing where class-level version is used for filtering matching_privacy_declarations, so rows are properly filtered before being handed to the TCF Experience builder. - Add temporary override_vendor_purposes config although this may need to be updated with other work that is going in. --- .fides/fides.toml | 1 + src/fides/api/models/sql_models.py | 130 +++++++- .../api/util/tcf/tcf_experience_contents.py | 22 +- src/fides/config/consent_settings.py | 11 + tests/conftest.py | 8 + .../ops/util/test_tcf_privacy_declarations.py | 277 ++++++++++++++++++ 6 files changed, 432 insertions(+), 17 deletions(-) create mode 100644 tests/ops/util/test_tcf_privacy_declarations.py diff --git a/.fides/fides.toml b/.fides/fides.toml index cc7fe748e0..f336635430 100644 --- a/.fides/fides.toml +++ b/.fides/fides.toml @@ -65,3 +65,4 @@ notification_service_type = "mailgun" [consent] tcf_enabled = false ac_enabled = false +override_vendor_purposes = false diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index a6fccc9e78..edd614283c 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -1,5 +1,5 @@ # type: ignore - +# pylint: disable=comparison-with-callable,no-member """ Contains all of the SqlAlchemy models for the Fides resources. """ @@ -7,7 +7,7 @@ from __future__ import annotations from enum import Enum as EnumType -from typing import Any, Dict, List, Optional, Set, Type, TypeVar +from typing import Any, Dict, List, Optional, Set, Type, TypeVar, Union from fideslang import MAPPED_PURPOSES_BY_DATA_USE from fideslang.gvl import MAPPED_PURPOSES, MappedPurpose @@ -24,14 +24,20 @@ Text, TypeDecorator, UniqueConstraint, + case, cast, + select, type_coerce, ) from sqlalchemy.dialects.postgresql import ARRAY, BIGINT, BYTEA +from sqlalchemy.engine import Row from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import Session, relationship +from sqlalchemy.orm.attributes import InstrumentedAttribute from sqlalchemy.sql import func +from sqlalchemy.sql.elements import Case +from sqlalchemy.sql.selectable import ScalarSelect from sqlalchemy.sql.sqltypes import DateTime from typing_extensions import Protocol, runtime_checkable @@ -41,10 +47,18 @@ from fides.api.models.client import ClientDetail from fides.api.models.fides_user import FidesUser from fides.api.models.fides_user_permissions import FidesUserPermissions +from fides.api.models.tcf_publisher_overrides import TCFPublisherOverrides from fides.config import get_config CONFIG = get_config() +# Mapping of data uses to *Purposes* not Special Purposes +MAPPED_PURPOSES_ONLY_BY_DATA_USE: Dict[str, MappedPurpose] = { + data_use: purpose + for data_use, purpose in MAPPED_PURPOSES_BY_DATA_USE.items() + if purpose in MAPPED_PURPOSES.values() +} + class FidesBase(FideslibBase): """ @@ -519,20 +533,114 @@ def create( @hybrid_property def purpose(self) -> Optional[int]: - """Return the Purpose ID (not Special Purpose ID) if the data use maps to a Purpose - - If data use maps to a Special Purpose, None is returned here. This is for Purposes only. - """ - mapped_purpose: Optional[MappedPurpose] = MAPPED_PURPOSES_BY_DATA_USE.get( + """Returns the instance-level TCF Purpose.""" + mapped_purpose: Optional[MappedPurpose] = MAPPED_PURPOSES_ONLY_BY_DATA_USE.get( self.data_use ) - if not mapped_purpose: + return mapped_purpose.id if mapped_purpose else None + + @purpose.expression + def purpose(cls) -> Case: + """Returns the class-level TCF Purpose""" + return case( + [ + (cls.data_use == data_use, purpose.id) + for data_use, purpose in MAPPED_PURPOSES_ONLY_BY_DATA_USE.items() + ], + else_=None, + ) + + @hybrid_property + def _publisher_override_legal_basis_join(self) -> Optional[str]: + """Returns the instance-level overridden required legal basis""" + db: Session = Session.object_session(self) + required_legal_basis: Optional[Row] = ( + db.query(TCFPublisherOverrides.required_legal_basis) + .filter(TCFPublisherOverrides.purpose == self.purpose) + .first() + ) + return required_legal_basis[0] if required_legal_basis else None + + @_publisher_override_legal_basis_join.expression + def _publisher_override_legal_basis_join(cls) -> ScalarSelect: + """Returns the class-level overridden required legal basis""" + return ( + select([TCFPublisherOverrides.required_legal_basis]) + .where(TCFPublisherOverrides.purpose == cls.purpose) + .as_scalar() + ) + + @hybrid_property + def _publisher_override_is_included_join(self) -> Optional[bool]: + """Returns the instance-level indication of whether the purpose should be included""" + db: Session = Session.object_session(self) + is_included: Optional[Row] = ( + db.query(TCFPublisherOverrides.is_included) + .filter(TCFPublisherOverrides.purpose == self.purpose) + .first() + ) + return is_included[0] if is_included else None + + @_publisher_override_is_included_join.expression + def _publisher_override_is_included_join(cls) -> ScalarSelect: + """Returns the class-level indication of whether the purpose should be included""" + return ( + select([TCFPublisherOverrides.is_included]) + .where(TCFPublisherOverrides.purpose == cls.purpose) + .as_scalar() + ) + + @hybrid_property + def overridden_legal_basis_for_processing(self) -> Optional[str]: + """ + Instance-level override of the legal basis for processing based on + publisher preferences. + """ + if not ( + CONFIG.consent.override_vendor_purposes + and self.flexible_legal_basis_for_processing + ): + return self.legal_basis_for_processing + + if self._publisher_override_is_included_join is False: + # Overriding to False to match behavior of class-level override. + # Class-level override of legal basis to None removes Privacy Declaration + # from Experience return None return ( - mapped_purpose.id - if MAPPED_PURPOSES.get(mapped_purpose.id) == mapped_purpose - else None + self._publisher_override_legal_basis_join + if self._publisher_override_legal_basis_join # pylint: disable=using-constant-test + else self.legal_basis_for_processing + ) + + @overridden_legal_basis_for_processing.expression + def overridden_legal_basis_for_processing( + cls, + ) -> Union[InstrumentedAttribute, Case]: + """ + Class-level override of the legal basis for processing based on + publisher preferences. + """ + if not CONFIG.consent.override_vendor_purposes: + return cls.legal_basis_for_processing + + return case( + [ + ( + cls.flexible_legal_basis_for_processing.is_(False), + cls.legal_basis_for_processing, + ), + ( + cls._publisher_override_is_included_join.is_(False), + None, + ), + ( + cls._publisher_override_legal_basis_join.is_(None), + cls.legal_basis_for_processing, + ), + ], + else_=cls._publisher_override_legal_basis_join, ) diff --git a/src/fides/api/util/tcf/tcf_experience_contents.py b/src/fides/api/util/tcf/tcf_experience_contents.py index c5213a5852..03d10845ec 100644 --- a/src/fides/api/util/tcf/tcf_experience_contents.py +++ b/src/fides/api/util/tcf/tcf_experience_contents.py @@ -57,10 +57,11 @@ not_(System.vendor_id.startswith(AC_PREFIX)), System.vendor_id.is_(None) ) CONSENT_LEGAL_BASIS_FILTER: BinaryExpression = ( - PrivacyDeclaration.legal_basis_for_processing == LegalBasisForProcessingEnum.CONSENT + PrivacyDeclaration.overridden_legal_basis_for_processing + == LegalBasisForProcessingEnum.CONSENT ) LEGITIMATE_INTEREST_LEGAL_BASIS_FILTER: BinaryExpression = ( - PrivacyDeclaration.legal_basis_for_processing + PrivacyDeclaration.overridden_legal_basis_for_processing == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST ) @@ -151,7 +152,10 @@ class TCFExperienceContents( def get_matching_privacy_declarations(db: Session) -> Query: """Returns flattened system/privacy declaration records where we have a matching gvl data use AND the - legal basis for processing is "Consent" or "Legitimate interests" + overridden legal basis for processing is "Consent" or "Legitimate interests". + + IMPORTANT - We are filtering against the "overridden_legal_basis_for_processing", not the defined "legal_basis_for_processing", + which takes into account potential Fides-wide publisher overrides. Only systems that meet this criteria should show up in the TCF overlay. """ @@ -171,22 +175,28 @@ def get_matching_privacy_declarations(db: Session) -> Query: System.privacy_policy.label("system_privacy_policy"), System.vendor_id, PrivacyDeclaration.data_use, - PrivacyDeclaration.legal_basis_for_processing, + PrivacyDeclaration.overridden_legal_basis_for_processing.label( # pylint: disable=no-member + "legal_basis_for_processing" + ), PrivacyDeclaration.features, PrivacyDeclaration.retention_period, PrivacyDeclaration.flexible_legal_basis_for_processing, + PrivacyDeclaration.purpose, + PrivacyDeclaration.legal_basis_for_processing.label( + "original_legal_basis_for_processing" + ), ) .outerjoin(PrivacyDeclaration, System.id == PrivacyDeclaration.system_id) .filter( or_( and_( GVL_DATA_USE_FILTER, - PrivacyDeclaration.legal_basis_for_processing + PrivacyDeclaration.overridden_legal_basis_for_processing == LegalBasisForProcessingEnum.CONSENT, ), and_( GVL_DATA_USE_FILTER, - PrivacyDeclaration.legal_basis_for_processing + PrivacyDeclaration.overridden_legal_basis_for_processing == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST, NOT_AC_SYSTEM_FILTER, ), diff --git a/src/fides/config/consent_settings.py b/src/fides/config/consent_settings.py index 877139659d..a16971b20e 100644 --- a/src/fides/config/consent_settings.py +++ b/src/fides/config/consent_settings.py @@ -15,6 +15,11 @@ class ConsentSettings(FidesSettings): default=False, description="Toggle whether Google AC Mode is enabled." ) + override_vendor_purposes: bool = Field( + default=False, + description="Whether or not vendor purposes can be globally overridden.", + ) + class Config: env_prefix = "FIDES__CONSENT__" @@ -23,8 +28,14 @@ def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: """AC mode only works if TCF mode is also enabled""" tcf_mode = values.get("tcf_enabled") ac_mode = values.get("ac_enabled") + override_vendor_purposes = values.get("override_vendor_purposes") if ac_mode and not tcf_mode: raise ValueError("AC cannot be enabled unless TCF mode is also enabled.") + if override_vendor_purposes and not tcf_mode: + raise ValueError( + "Override vendor purposes cannot be true unless TCF mode is also enabled." + ) + return values diff --git a/tests/conftest.py b/tests/conftest.py index 8a9642bc2a..dd27da6aad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -185,6 +185,14 @@ def enable_ac(config): config.consent.ac_enabled = False +@pytest.fixture(scope="function") +def enable_override_vendor_purposes(config): + assert config.test_mode + config.consent.override_vendor_purposes = True + yield config + config.consent.override_vendor_purposes = False + + @pytest.fixture def loguru_caplog(caplog): handler_id = logger.add(caplog.handler, format="{message}") diff --git a/tests/ops/util/test_tcf_privacy_declarations.py b/tests/ops/util/test_tcf_privacy_declarations.py new file mode 100644 index 0000000000..bd028eb0e7 --- /dev/null +++ b/tests/ops/util/test_tcf_privacy_declarations.py @@ -0,0 +1,277 @@ +from uuid import uuid4 + +import pytest + +from fides.api.models.sql_models import PrivacyDeclaration +from fides.api.models.tcf_publisher_overrides import TCFPublisherOverrides +from fides.api.util.tcf.tcf_experience_contents import get_matching_privacy_declarations + + +class TestPrivacyDeclarationInstanceLevelHybridProperties: + """Test some instance-level hybrid properties defined on the Privacy Declaration + related to Publisher Overrides""" + + def test_privacy_declaration_enable_override_is_false(self, system, db): + """Enable override is false so overridden legal basis is going to default + to the defined legal basis""" + pd = PrivacyDeclaration( + name=f"declaration-name-{uuid4()}", + data_categories=[], + data_use="analytics.reporting.campaign_insights", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ingress=None, + egress=None, + legal_basis_for_processing="Consent", + system_id=system.id, + ).save(db=db) + + assert pd.purpose == 9 + assert pd._publisher_override_legal_basis_join is None + assert pd._publisher_override_is_included_join is None + assert pd.overridden_legal_basis_for_processing == "Consent" + + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + def test_enable_override_is_true_but_no_matching_purpose(self, system, db): + """Privacy Declaration has Special Purpose not Purpose, so no overrides applicable""" + pd = PrivacyDeclaration( + name=f"declaration-name-{uuid4()}", + data_categories=[], + data_use="essential.fraud_detection", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ingress=None, + egress=None, + legal_basis_for_processing="Consent", + system_id=system.id, + ).save(db) + + assert pd.purpose is None + assert pd._publisher_override_legal_basis_join is None + assert pd._publisher_override_is_included_join is None + assert pd.overridden_legal_basis_for_processing == "Consent" + + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + def test_enable_override_is_true_but_purpose_is_excluded(self, db, system): + """Purpose is overridden as excluded, so legal basis returns as None, to match + class-wide override""" + TCFPublisherOverrides.create( + db, + data={ + "purpose": 9, + "is_included": False, + }, + ) + + pd = PrivacyDeclaration( + name="declaration-name", + data_categories=[], + data_use="analytics.reporting.campaign_insights", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ingress=None, + egress=None, + flexible_legal_basis_for_processing=True, + legal_basis_for_processing="Consent", + system_id=system.id, + ).save(db=db) + + assert pd.purpose == 9 + assert pd._publisher_override_legal_basis_join is None + assert pd._publisher_override_is_included_join is False + assert pd.overridden_legal_basis_for_processing is None + + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + def test_publisher_override_defined_but_no_required_legal_basis_specified( + self, db, system + ): + """Purpose override is defined, but no legal basis override""" + TCFPublisherOverrides.create( + db, + data={ + "purpose": 9, + "is_included": True, + }, + ) + + pd = PrivacyDeclaration( + name="declaration-name", + data_categories=[], + data_use="analytics.reporting.campaign_insights", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ingress=None, + egress=None, + flexible_legal_basis_for_processing=True, + legal_basis_for_processing="Consent", + system_id=system.id, + ).save(db=db) + + assert pd.purpose == 9 + assert pd._publisher_override_legal_basis_join is None + assert pd._publisher_override_is_included_join is True + assert pd.overridden_legal_basis_for_processing == "Consent" + + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + def test_publisher_override_defined_with_required_legal_basis_specified( + self, db, system + ): + """Purpose override is defined, but no legal basis override""" + TCFPublisherOverrides.create( + db, + data={ + "purpose": 9, + "is_included": True, + "required_legal_basis": "Legitimate interests", + }, + ) + + pd = PrivacyDeclaration( + name="declaration-name", + data_categories=[], + data_use="analytics.reporting.campaign_insights", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ingress=None, + egress=None, + flexible_legal_basis_for_processing=True, + legal_basis_for_processing="Consent", + system_id=system.id, + ).save(db=db) + + assert pd.purpose == 9 + assert pd._publisher_override_legal_basis_join == "Legitimate interests" + assert pd._publisher_override_is_included_join is True + assert pd.overridden_legal_basis_for_processing == "Legitimate interests" + + +class TestMatchingPrivacyDeclarations: + """Tests matching privacy declarations returned that are the basis of the TCF Experience and the "relevant_systems" + that are saved for consent reporting + """ + + @pytest.mark.usefixtures( + "emerse_system", + ) + def test_get_matching_privacy_declarations_enable_purpose_override_is_false( + self, emerse_system, db + ): + declarations = get_matching_privacy_declarations(db) + + assert declarations.count() == 13 + + mapping = { + declaration.data_use: declaration.purpose for declaration in declarations + } + + # marketing.advertising.serving, essential.service.security, essential.fraud_detection map to special purposes, not purposes + assert mapping == { + "marketing.advertising.serving": None, + "essential.service.security": None, + "essential.fraud_detection": None, + "analytics.reporting.campaign_insights": 9, + "analytics.reporting.content_performance": 8, + "analytics.reporting.ad_performance": 7, + "marketing.advertising.frequency_capping": 2, + "marketing.advertising.first_party.contextual": 2, + "marketing.advertising.negative_targeting": 2, + "marketing.advertising.first_party.targeted": 4, + "marketing.advertising.third_party.targeted": 4, + "marketing.advertising.profiling": 3, + "functional.storage": 1, + } + + @pytest.mark.usefixtures("emerse_system", "enable_override_vendor_purposes") + def test_privacy_declaration_publisher_overrides( + self, + db, + ): + """Define some purpose legal basis overrides and check their effects on what is returned in the Privacy Declaration query""" + + # Defined legal basis is also Consent for purpose 1 on Emerse. + # Publisher override matches. + TCFPublisherOverrides.create( + db, + data={"purpose": 1, "is_included": True, "required_legal_basis": "Consent"}, + ) + + # Defined legal basis is Legitimate Interests for purpose 2 on Emerse. + # Here, Purpose 2 is specified to be excluded. + TCFPublisherOverrides.create( + db, + data={ + "purpose": 2, + "is_included": False, + }, + ) + + # Defined legal basis is Consent for purpose 3 on Emerse. + # No legal basis override is defined. + TCFPublisherOverrides.create( + db, + data={ + "purpose": 3, + "is_included": True, + "required_legal_basis": None, + }, + ) + + # Defined legal basis is Consent for purpose 4 on Emerse. + # Override here has a different legal basis + TCFPublisherOverrides.create( + db, + data={ + "purpose": 4, + "is_included": True, + "required_legal_basis": "Legitimate interests", + }, + ) + + declarations = get_matching_privacy_declarations(db) + + legal_basis_overrides = { + declaration.purpose: declaration.legal_basis_for_processing + for declaration in declarations + if declaration.purpose + } + + # Purpose 2 has been removed altogether and Purpose 4 Legal Basis + # has been overridden to Legitimate Interests legal basis + assert legal_basis_overrides == { + 9: "Legitimate interests", + 8: "Legitimate interests", + 7: "Legitimate interests", + 4: "Legitimate interests", + 3: "Consent", + 1: "Consent", + } + + original_legal_basis = { + declaration.purpose: declaration.original_legal_basis_for_processing + for declaration in declarations + if declaration.purpose + } + assert original_legal_basis == { + 9: "Legitimate interests", + 8: "Legitimate interests", + 7: "Legitimate interests", + 4: "Consent", + 3: "Consent", + 1: "Consent", + } + + # The three declarations on Emerse with data uses mapping to Purpose 2 have been excluded + assert declarations.count() == 10 From ff4b6e61c09b181e9e8ff0b7ae03c3c10b6aa85a Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 28 Nov 2023 11:44:11 -0600 Subject: [PATCH 03/15] Rename tcf publisher overrides table. --- .../5225ea4de265_tcf_publisher_overrides.py | 10 +++++----- src/fides/api/db/base.py | 2 +- src/fides/api/models/sql_models.py | 18 +++++++++--------- .../api/models/tcf_publisher_overrides.py | 7 ++++++- .../ops/util/test_tcf_privacy_declarations.py | 16 ++++++++-------- 5 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py b/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py index 98ef26edf7..3a1bee3274 100644 --- a/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py +++ b/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py @@ -17,7 +17,7 @@ def upgrade(): op.create_table( - "tcfpublisheroverrides", + "tcf_publisher_overrides", sa.Column("id", sa.String(length=255), nullable=False), sa.Column( "created_at", @@ -37,8 +37,8 @@ def upgrade(): sa.PrimaryKeyConstraint("id"), ) op.create_index( - op.f("ix_tcfpublisheroverrides_id"), - "tcfpublisheroverrides", + op.f("ix_tcf_publisher_overrides_id"), + "tcf_publisher_overrides", ["id"], unique=False, ) @@ -46,6 +46,6 @@ def upgrade(): def downgrade(): op.drop_index( - op.f("ix_tcfpublisheroverrides_id"), table_name="tcfpublisheroverrides" + op.f("ix_tcf_publisher_overrides_id"), table_name="tcf_publisher_overrides" ) - op.drop_table("tcfpublisheroverrides") + op.drop_table("tcf_publisher_overrides") diff --git a/src/fides/api/db/base.py b/src/fides/api/db/base.py index b27d20c258..1b29fe9fb3 100644 --- a/src/fides/api/db/base.py +++ b/src/fides/api/db/base.py @@ -39,4 +39,4 @@ from fides.api.models.system_compass_sync import SystemCompassSync from fides.api.models.system_history import SystemHistory from fides.api.models.system_manager import SystemManager -from fides.api.models.tcf_publisher_overrides import TCFPublisherOverrides +from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index edd614283c..d290fa51d8 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -47,7 +47,7 @@ from fides.api.models.client import ClientDetail from fides.api.models.fides_user import FidesUser from fides.api.models.fides_user_permissions import FidesUserPermissions -from fides.api.models.tcf_publisher_overrides import TCFPublisherOverrides +from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride from fides.config import get_config CONFIG = get_config() @@ -555,8 +555,8 @@ def _publisher_override_legal_basis_join(self) -> Optional[str]: """Returns the instance-level overridden required legal basis""" db: Session = Session.object_session(self) required_legal_basis: Optional[Row] = ( - db.query(TCFPublisherOverrides.required_legal_basis) - .filter(TCFPublisherOverrides.purpose == self.purpose) + db.query(TCFPublisherOverride.required_legal_basis) + .filter(TCFPublisherOverride.purpose == self.purpose) .first() ) return required_legal_basis[0] if required_legal_basis else None @@ -565,8 +565,8 @@ def _publisher_override_legal_basis_join(self) -> Optional[str]: def _publisher_override_legal_basis_join(cls) -> ScalarSelect: """Returns the class-level overridden required legal basis""" return ( - select([TCFPublisherOverrides.required_legal_basis]) - .where(TCFPublisherOverrides.purpose == cls.purpose) + select([TCFPublisherOverride.required_legal_basis]) + .where(TCFPublisherOverride.purpose == cls.purpose) .as_scalar() ) @@ -575,8 +575,8 @@ def _publisher_override_is_included_join(self) -> Optional[bool]: """Returns the instance-level indication of whether the purpose should be included""" db: Session = Session.object_session(self) is_included: Optional[Row] = ( - db.query(TCFPublisherOverrides.is_included) - .filter(TCFPublisherOverrides.purpose == self.purpose) + db.query(TCFPublisherOverride.is_included) + .filter(TCFPublisherOverride.purpose == self.purpose) .first() ) return is_included[0] if is_included else None @@ -585,8 +585,8 @@ def _publisher_override_is_included_join(self) -> Optional[bool]: def _publisher_override_is_included_join(cls) -> ScalarSelect: """Returns the class-level indication of whether the purpose should be included""" return ( - select([TCFPublisherOverrides.is_included]) - .where(TCFPublisherOverrides.purpose == cls.purpose) + select([TCFPublisherOverride.is_included]) + .where(TCFPublisherOverride.purpose == cls.purpose) .as_scalar() ) diff --git a/src/fides/api/models/tcf_publisher_overrides.py b/src/fides/api/models/tcf_publisher_overrides.py index 20e8d401a4..be2fa420c8 100644 --- a/src/fides/api/models/tcf_publisher_overrides.py +++ b/src/fides/api/models/tcf_publisher_overrides.py @@ -1,13 +1,18 @@ from sqlalchemy import Boolean, Column, Integer, String +from sqlalchemy.orm import declared_attr from fides.api.db.base_class import Base -class TCFPublisherOverrides(Base): +class TCFPublisherOverride(Base): """ Stores TCF Publisher Overrides """ + @declared_attr + def __tablename__(self) -> str: + return "tcf_publisher_overrides" + purpose = Column(Integer, nullable=False) is_included = Column(Boolean, server_default="t", default=True) required_legal_basis = Column(String) diff --git a/tests/ops/util/test_tcf_privacy_declarations.py b/tests/ops/util/test_tcf_privacy_declarations.py index bd028eb0e7..7b19dd7fb9 100644 --- a/tests/ops/util/test_tcf_privacy_declarations.py +++ b/tests/ops/util/test_tcf_privacy_declarations.py @@ -3,7 +3,7 @@ import pytest from fides.api.models.sql_models import PrivacyDeclaration -from fides.api.models.tcf_publisher_overrides import TCFPublisherOverrides +from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride from fides.api.util.tcf.tcf_experience_contents import get_matching_privacy_declarations @@ -61,7 +61,7 @@ def test_enable_override_is_true_but_no_matching_purpose(self, system, db): def test_enable_override_is_true_but_purpose_is_excluded(self, db, system): """Purpose is overridden as excluded, so legal basis returns as None, to match class-wide override""" - TCFPublisherOverrides.create( + TCFPublisherOverride.create( db, data={ "purpose": 9, @@ -95,7 +95,7 @@ def test_publisher_override_defined_but_no_required_legal_basis_specified( self, db, system ): """Purpose override is defined, but no legal basis override""" - TCFPublisherOverrides.create( + TCFPublisherOverride.create( db, data={ "purpose": 9, @@ -129,7 +129,7 @@ def test_publisher_override_defined_with_required_legal_basis_specified( self, db, system ): """Purpose override is defined, but no legal basis override""" - TCFPublisherOverrides.create( + TCFPublisherOverride.create( db, data={ "purpose": 9, @@ -203,14 +203,14 @@ def test_privacy_declaration_publisher_overrides( # Defined legal basis is also Consent for purpose 1 on Emerse. # Publisher override matches. - TCFPublisherOverrides.create( + TCFPublisherOverride.create( db, data={"purpose": 1, "is_included": True, "required_legal_basis": "Consent"}, ) # Defined legal basis is Legitimate Interests for purpose 2 on Emerse. # Here, Purpose 2 is specified to be excluded. - TCFPublisherOverrides.create( + TCFPublisherOverride.create( db, data={ "purpose": 2, @@ -220,7 +220,7 @@ def test_privacy_declaration_publisher_overrides( # Defined legal basis is Consent for purpose 3 on Emerse. # No legal basis override is defined. - TCFPublisherOverrides.create( + TCFPublisherOverride.create( db, data={ "purpose": 3, @@ -231,7 +231,7 @@ def test_privacy_declaration_publisher_overrides( # Defined legal basis is Consent for purpose 4 on Emerse. # Override here has a different legal basis - TCFPublisherOverrides.create( + TCFPublisherOverride.create( db, data={ "purpose": 4, From 4d3c1b8a6955a9dc4694f7fd5958a4de8f292408 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 28 Nov 2023 11:49:46 -0600 Subject: [PATCH 04/15] Add a unique purpose constraint to tcf_publisher_overrides. --- .../versions/5225ea4de265_tcf_publisher_overrides.py | 4 ++++ src/fides/api/models/tcf_publisher_overrides.py | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py b/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py index 3a1bee3274..aa11c5edbf 100644 --- a/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py +++ b/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py @@ -43,6 +43,10 @@ def upgrade(): unique=False, ) + op.create_unique_constraint( + "purpose_constraint", "tcf_publisher_overrides", ["purpose"] + ) + def downgrade(): op.drop_index( diff --git a/src/fides/api/models/tcf_publisher_overrides.py b/src/fides/api/models/tcf_publisher_overrides.py index be2fa420c8..a47444618e 100644 --- a/src/fides/api/models/tcf_publisher_overrides.py +++ b/src/fides/api/models/tcf_publisher_overrides.py @@ -1,4 +1,4 @@ -from sqlalchemy import Boolean, Column, Integer, String +from sqlalchemy import Boolean, Column, Integer, String, UniqueConstraint from sqlalchemy.orm import declared_attr from fides.api.db.base_class import Base @@ -7,6 +7,9 @@ class TCFPublisherOverride(Base): """ Stores TCF Publisher Overrides + + Allows a customer to override Fides-wide which purposes show up in the TCF Experience, and + specify a global legal basis. """ @declared_attr @@ -16,3 +19,5 @@ def __tablename__(self) -> str: purpose = Column(Integer, nullable=False) is_included = Column(Boolean, server_default="t", default=True) required_legal_basis = Column(String) + + UniqueConstraint("purpose") From 8e4235ec35f599fd38fb0ec92bba08b791e21c0d Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 28 Nov 2023 12:08:05 -0600 Subject: [PATCH 05/15] Load default TCF publisher override resources on startup, one for each purpose. The default configuration does not actually override any purposes or legal bases. These also aren't used at all if the the config variable override_vendor_purposes is False. --- src/fides/api/app_setup.py | 15 ++++++++++++ .../api/models/tcf_publisher_overrides.py | 6 ++--- src/fides/api/util/consent_util.py | 23 +++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/fides/api/app_setup.py b/src/fides/api/app_setup.py index 51f22c989f..c9444a6880 100644 --- a/src/fides/api/app_setup.py +++ b/src/fides/api/app_setup.py @@ -41,6 +41,7 @@ from fides.api.service.saas_request.override_implementations import * from fides.api.util.cache import get_cache from fides.api.util.consent_util import ( + create_default_tcf_publisher_overrides_on_startup, create_tcf_experiences_on_startup, load_default_experience_configs_on_startup, load_default_notices_on_startup, @@ -204,7 +205,9 @@ async def run_database_startup(app: FastAPI) -> None: # Default notices subject to change, so preventing these from # loading in test mode to avoid interfering with unit tests. load_default_privacy_notices() + # Similarly avoiding loading other consent out-of-the-box resources to avoid interfering with unit tests load_tcf_experiences() + load_tcf_publisher_overrides() db.close() @@ -255,3 +258,15 @@ def load_tcf_experiences() -> None: logger.error("Skipping loading TCF Overlay Experiences: {}", str(e)) finally: db.close() + + +def load_tcf_publisher_overrides() -> None: + """Load default tcf publisher overrides""" + logger.info("Loading default TCF Publisher Overrides") + try: + db = get_api_session() + create_default_tcf_publisher_overrides_on_startup(db) + except Exception as e: + logger.error("Skipping loading TCF Publisher Overrides: {}", str(e)) + finally: + db.close() diff --git a/src/fides/api/models/tcf_publisher_overrides.py b/src/fides/api/models/tcf_publisher_overrides.py index a47444618e..b326dd4ceb 100644 --- a/src/fides/api/models/tcf_publisher_overrides.py +++ b/src/fides/api/models/tcf_publisher_overrides.py @@ -1,5 +1,5 @@ from sqlalchemy import Boolean, Column, Integer, String, UniqueConstraint -from sqlalchemy.orm import declared_attr +from sqlalchemy.ext.declarative import declared_attr from fides.api.db.base_class import Base @@ -8,8 +8,8 @@ class TCFPublisherOverride(Base): """ Stores TCF Publisher Overrides - Allows a customer to override Fides-wide which purposes show up in the TCF Experience, and - specify a global legal basis. + Allows a customer to override Fides-wide which Purposes show up in the TCF Experience, and + specify a global legal basis for that Purpose. """ @declared_attr diff --git a/src/fides/api/util/consent_util.py b/src/fides/api/util/consent_util.py index bc0333fe42..65ad4910bd 100644 --- a/src/fides/api/util/consent_util.py +++ b/src/fides/api/util/consent_util.py @@ -34,6 +34,7 @@ ProvidedIdentityType, ) from fides.api.models.sql_models import DataUse, System # type: ignore[attr-defined] +from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride from fides.api.schemas.privacy_experience import ExperienceConfigCreateWithId from fides.api.schemas.privacy_notice import PrivacyNoticeCreation, PrivacyNoticeWithId from fides.api.schemas.redis_cache import Identity @@ -671,3 +672,25 @@ def create_tcf_experiences_on_startup(db: Session) -> List[PrivacyExperience]: ) return experiences_created + + +def create_default_tcf_publisher_overrides_on_startup( + db: Session, +) -> List[TCFPublisherOverride]: + """On startup, load default Publisher Overrides, one for each purpose, with a default of is_included=True + and no legal basis override""" + publisher_overrides_created: List[TCFPublisherOverride] = [] + + for purpose_id in range(1, 12): + if ( + not db.query(TCFPublisherOverride) + .filter(TCFPublisherOverride.purpose == purpose_id) + .first() + ): + publisher_overrides_created.append( + TCFPublisherOverride.create( + db, data={"purpose": purpose_id, "is_included": True} + ) + ) + + return publisher_overrides_created From a6c099ff1c2e3957aa3d11818f1f938e0b79110a Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 28 Nov 2023 17:26:34 -0600 Subject: [PATCH 06/15] WIP Refactor so instance-level hybrid properties on PrivacyDeclarations use async sessions so they can be accessed via existing System API routes. Current strategy is to stash these on the declaration under a new key since declaring the hybrid_property directly on the Privacy Declaration Schema causes coroutine issues when the PrivacyDeclaration is serialized under "serialize_response" --- src/fides/api/api/v1/endpoints/system.py | 35 ++- src/fides/api/models/sql_models.py | 47 +-- src/fides/api/schemas/system.py | 4 + tests/ctl/core/test_api.py | 279 +++++++++++++++++- .../ops/util/test_tcf_privacy_declarations.py | 156 +--------- 5 files changed, 338 insertions(+), 183 deletions(-) diff --git a/src/fides/api/api/v1/endpoints/system.py b/src/fides/api/api/v1/endpoints/system.py index 59fa3be6a7..329cd79f5b 100644 --- a/src/fides/api/api/v1/endpoints/system.py +++ b/src/fides/api/api/v1/endpoints/system.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union from fastapi import Depends, HTTPException, Response, Security from fastapi_pagination import Page, Params @@ -258,6 +258,10 @@ async def update( updated_system, _ = await update_system( resource, db, current_user.id if current_user else None ) + await supplement_privacy_declaration_response_with_legal_basis_override( + updated_system + ) + return updated_system @@ -353,7 +357,11 @@ async def create( Override `System` create/POST to handle `.privacy_declarations` defined inline, for backward compatibility and ease of use for API users. """ - return await create_system(resource, db, current_user.id if current_user else None) + created = await create_system( + resource, db, current_user.id if current_user else None + ) + await supplement_privacy_declaration_response_with_legal_basis_override(created) + return created @SYSTEM_ROUTER.get( @@ -371,7 +379,10 @@ async def ls( # pylint: disable=invalid-name db: AsyncSession = Depends(get_async_db), ) -> List: """Get a list of all of the resources of this type.""" - return await list_resource(System, db) + systems = await list_resource(System, db) + for system in systems: + await supplement_privacy_declaration_response_with_legal_basis_override(system) + return systems @SYSTEM_ROUTER.get( @@ -389,7 +400,10 @@ async def get( db: AsyncSession = Depends(get_async_db), ) -> Dict: """Get a resource by its fides_key.""" - return await get_resource_with_custom_fields(System, fides_key, db) + + resp = await get_resource_with_custom_fields(System, fides_key, db) + await supplement_privacy_declaration_response_with_legal_basis_override(resp) + return resp @SYSTEM_CONNECTION_INSTANTIATE_ROUTER.post( @@ -412,3 +426,16 @@ def instantiate_connection_from_template( system = get_system(db, fides_key) return instantiate_connection(db, saas_connector_type, template_values, system) + + +async def supplement_privacy_declaration_response_with_legal_basis_override(resp: Union[Dict, System]) -> None: + """At runtime, adds a "legal_basis_for_processing_override" to each PrivacyDeclaration""" + + for privacy_declaration in ( + resp.get("privacy_declarations") + if isinstance(resp, Dict) + else resp.privacy_declarations + ): + privacy_declaration.legal_basis_for_processing_override = ( + await privacy_declaration.overridden_legal_basis_for_processing + ) diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index d290fa51d8..7c88198516 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -32,10 +32,11 @@ from sqlalchemy.dialects.postgresql import ARRAY, BIGINT, BYTEA from sqlalchemy.engine import Row from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession, async_object_session from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy.orm import Session, relationship +from sqlalchemy.orm import Session, object_session, relationship from sqlalchemy.orm.attributes import InstrumentedAttribute -from sqlalchemy.sql import func +from sqlalchemy.sql import func, Select from sqlalchemy.sql.elements import Case from sqlalchemy.sql.selectable import ScalarSelect from sqlalchemy.sql.sqltypes import DateTime @@ -551,15 +552,15 @@ def purpose(cls) -> Case: ) @hybrid_property - def _publisher_override_legal_basis_join(self) -> Optional[str]: + async def _publisher_override_legal_basis_join(self) -> Optional[str]: """Returns the instance-level overridden required legal basis""" - db: Session = Session.object_session(self) - required_legal_basis: Optional[Row] = ( - db.query(TCFPublisherOverride.required_legal_basis) - .filter(TCFPublisherOverride.purpose == self.purpose) - .first() + query: Select = select([TCFPublisherOverride.required_legal_basis]).where( + TCFPublisherOverride.purpose == self.purpose ) - return required_legal_basis[0] if required_legal_basis else None + async_session: AsyncSession = async_object_session(self) + async with async_session.begin(): + result = await async_session.execute(query) + return result.scalars().first() @_publisher_override_legal_basis_join.expression def _publisher_override_legal_basis_join(cls) -> ScalarSelect: @@ -571,15 +572,15 @@ def _publisher_override_legal_basis_join(cls) -> ScalarSelect: ) @hybrid_property - def _publisher_override_is_included_join(self) -> Optional[bool]: + async def _publisher_override_is_included_join(self) -> Optional[bool]: """Returns the instance-level indication of whether the purpose should be included""" - db: Session = Session.object_session(self) - is_included: Optional[Row] = ( - db.query(TCFPublisherOverride.is_included) - .filter(TCFPublisherOverride.purpose == self.purpose) - .first() + query: Select = select([TCFPublisherOverride.is_included]).where( + TCFPublisherOverride.purpose == self.purpose ) - return is_included[0] if is_included else None + async_session: AsyncSession = async_object_session(self) + async with async_session.begin(): + result = await async_session.execute(query) + return result.scalars().first() @_publisher_override_is_included_join.expression def _publisher_override_is_included_join(cls) -> ScalarSelect: @@ -591,7 +592,7 @@ def _publisher_override_is_included_join(cls) -> ScalarSelect: ) @hybrid_property - def overridden_legal_basis_for_processing(self) -> Optional[str]: + async def overridden_legal_basis_for_processing(self) -> Optional[str]: """ Instance-level override of the legal basis for processing based on publisher preferences. @@ -602,15 +603,21 @@ def overridden_legal_basis_for_processing(self) -> Optional[str]: ): return self.legal_basis_for_processing - if self._publisher_override_is_included_join is False: + is_included: Optional[bool] = await self._publisher_override_is_included_join + + if is_included is False: # Overriding to False to match behavior of class-level override. # Class-level override of legal basis to None removes Privacy Declaration # from Experience return None + overridden_legal_basis: Optional[str] = ( + await self._publisher_override_legal_basis_join + ) + return ( - self._publisher_override_legal_basis_join - if self._publisher_override_legal_basis_join # pylint: disable=using-constant-test + overridden_legal_basis + if overridden_legal_basis # pylint: disable=using-constant-test else self.legal_basis_for_processing ) diff --git a/src/fides/api/schemas/system.py b/src/fides/api/schemas/system.py index 278fc7ea76..ddae949a37 100644 --- a/src/fides/api/schemas/system.py +++ b/src/fides/api/schemas/system.py @@ -19,6 +19,10 @@ class PrivacyDeclarationResponse(PrivacyDeclaration): ) cookies: Optional[List[Cookies]] = [] + legal_basis_for_processing_override: Optional[str] = Field( + description="Global overrides for this purpose's legal basis for processing if applicable. Defaults to the legal_basis_for_processing otherwise." + ) + class BasicSystemResponse(System): """ diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index ba2c1357a5..bc9eadb35e 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -5,10 +5,12 @@ from datetime import datetime from json import loads from typing import Dict, List +from uuid import uuid4 import pytest import requests from fideslang import DEFAULT_TAXONOMY, model_list, models, parse +from fideslang.models import PrivacyDeclaration as PrivacyDeclarationSchema from fideslang.models import System as SystemSchema from pytest import MonkeyPatch from starlette.status import ( @@ -25,10 +27,12 @@ from fides.api.api.v1.endpoints import health from fides.api.db.crud import get_resource +from fides.api.db.system import create_system from fides.api.models.connectionconfig import ConnectionConfig from fides.api.models.datasetconfig import DatasetConfig from fides.api.models.sql_models import Dataset, PrivacyDeclaration, System from fides.api.models.system_history import SystemHistory +from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride from fides.api.oauth.roles import OWNER, VIEWER from fides.api.schemas.system import PrivacyDeclarationResponse, SystemResponse from fides.api.schemas.taxonomy_extensions import ( @@ -629,8 +633,8 @@ async def test_system_create( for i, decl in enumerate(system.privacy_declarations): for field in PrivacyDeclarationResponse.__fields__: - decl_val = getattr(decl, field) - if isinstance(decl_val, typing.Hashable): + decl_val = getattr(decl, field, None) + if hasattr(decl, field) and isinstance(decl_val, typing.Hashable): assert decl_val == json_results["privacy_declarations"][i][field] assert len(system.privacy_declarations) == 2 @@ -678,6 +682,10 @@ async def test_system_create( assert privacy_decl.dataset_references == ["another_system_reference"] assert privacy_decl.features == ["Link different devices"] assert privacy_decl.legal_basis_for_processing == "Public interest" + assert ( + await privacy_decl.overridden_legal_basis_for_processing + == "Public interest" + ) assert ( privacy_decl.impact_assessment_location == "https://www.example.com/impact_assessment_location" @@ -835,6 +843,10 @@ async def test_system_create_custom_metadata_saas_config( assert result.status_code == HTTP_201_CREATED assert result.json()["name"] == "Test System" assert len(result.json()["privacy_declarations"]) == 2 + assert { + decl["legal_basis_for_processing_override"] + for decl in result.json()["privacy_declarations"] + } == {"Public interest", None} assert result.json()["meta"] == { "saas_config": { "type": "stripe", @@ -996,6 +1008,70 @@ def test_data_stewards_included_in_response( assert "first_name" in steward assert "last_name" in steward + @pytest.mark.usefixtures("enable_override_vendor_purposes") + async def test_get_overridden_declaration_legal_basis( + self, db, test_config, async_session_temp + ): + resource = SystemSchema( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name=f"test_system_1_{uuid4()}", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="Collect data for content performance", + data_use="analytics.reporting.content_performance", + legal_basis_for_processing="Consent", + data_categories=["user"], + ) + ], + ) + + system = await create_system( + resource, async_session_temp, CONFIG.security.oauth_root_client_id + ) + + TCFPublisherOverride.create( + db, + data={ + "purpose": 8, + "is_included": True, + "required_legal_basis": "Legitimate interests", + }, + ) + + decl = next( + ( + declaration + for declaration in system.privacy_declarations + if declaration.data_use == "analytics.reporting.content_performance" + ), + None, + ) + assert decl.legal_basis_for_processing == "Consent" + assert ( + await decl.overridden_legal_basis_for_processing == "Legitimate interests" + ) + + result = _api.get( + url=test_config.cli.server_url, + headers=test_config.user.auth_header, + resource_type="system", + resource_id=system.fides_key, + ) + assert result.status_code == 200 + assert result.json()["fides_key"] == system.fides_key + assert ( + result.json()["privacy_declarations"][0]["legal_basis_for_processing"] + == "Consent" + ) + assert ( + result.json()["privacy_declarations"][0][ + "legal_basis_for_processing_override" + ] + == "Legitimate interests" + ) + @pytest.mark.unit class TestSystemUpdate: @@ -1593,8 +1669,8 @@ def test_system_update_dictionary_fields( for i, decl in enumerate(system.privacy_declarations): for field in PrivacyDeclarationResponse.__fields__: - decl_val = getattr(decl, field) - if isinstance(decl_val, typing.Hashable): + decl_val = getattr(decl, field, None) + if hasattr(decl, field) and isinstance(decl_val, typing.Hashable): assert decl_val == json_results["privacy_declarations"][i][field] def test_system_update_system_cookies( @@ -2592,3 +2668,198 @@ def test_trailing_slash(test_config: FidesConfig, endpoint_name: str) -> None: assert response.status_code == 200 response = requests.get(f"{url}/", headers=CONFIG.user.auth_header) assert response.status_code == 200 + + +class TestPrivacyDeclarationInstanceLevelHybridProperties: + """Test some instance-level hybrid properties defined on the Privacy Declaration + related to Publisher Overrides""" + + async def test_privacy_declaration_enable_override_is_false( + self, async_session_temp + ): + """Enable override is false so overridden legal basis is going to default + to the defined legal basis""" + resource = SystemSchema( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name=f"test_system_1_{uuid4()}", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="Collect data for content performance", + data_use="analytics.reporting.campaign_insights", + legal_basis_for_processing="Consent", + data_categories=["user"], + ) + ], + ) + + system = await create_system( + resource, async_session_temp, CONFIG.security.oauth_root_client_id + ) + pd = system.privacy_declarations[0] + + assert pd.purpose == 9 + assert await pd._publisher_override_legal_basis_join is None + assert await pd._publisher_override_is_included_join is None + assert await pd.overridden_legal_basis_for_processing == "Consent" + + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + async def test_enable_override_is_true_but_no_matching_purpose( + self, async_session_temp, db + ): + """Privacy Declaration has Special Purpose not Purpose, so no overrides applicable""" + resource = SystemSchema( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name=f"test_system_1_{uuid4()}", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="Collect data for content performance", + data_use="essential.fraud_detection", + legal_basis_for_processing="Consent", + data_categories=["user"], + ) + ], + ) + + system = await create_system( + resource, async_session_temp, CONFIG.security.oauth_root_client_id + ) + pd = system.privacy_declarations[0] + + assert pd.purpose is None + assert await pd._publisher_override_legal_basis_join is None + assert await pd._publisher_override_is_included_join is None + assert await pd.overridden_legal_basis_for_processing == "Consent" + + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + async def test_enable_override_is_true_but_purpose_is_excluded( + self, async_session_temp, db + ): + """Purpose is overridden as excluded, so legal basis returns as None, to match + class-wide override""" + resource = SystemSchema( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name=f"test_system_1_{uuid4()}", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="Collect data for content performance", + data_use="personalize.content.profiling", + legal_basis_for_processing="Consent", + data_categories=["user"], + ) + ], + ) + + system = await create_system( + resource, async_session_temp, CONFIG.security.oauth_root_client_id + ) + pd = system.privacy_declarations[0] + + constraint = TCFPublisherOverride.create( + db, + data={ + "purpose": 5, + "is_included": False, + }, + ) + + assert pd.purpose == 5 + assert await pd._publisher_override_legal_basis_join is None + assert await pd._publisher_override_is_included_join is False + assert await pd.overridden_legal_basis_for_processing is None + + constraint.delete(db) + + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + async def test_publisher_override_defined_but_no_required_legal_basis_specified( + self, db, async_session_temp + ): + """Purpose override is defined, but no legal basis override""" + resource = SystemSchema( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name=f"test_system_1_{uuid4()}", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="Collect data for content performance", + data_use="analytics.reporting.campaign_insights", + legal_basis_for_processing="Consent", + data_categories=["user"], + ) + ], + ) + + system = await create_system( + resource, async_session_temp, CONFIG.security.oauth_root_client_id + ) + pd = system.privacy_declarations[0] + + constraint = TCFPublisherOverride.create( + db, + data={ + "purpose": 9, + "is_included": True, + }, + ) + + assert pd.purpose == 9 + assert await pd._publisher_override_legal_basis_join is None + assert await pd._publisher_override_is_included_join is True + assert await pd.overridden_legal_basis_for_processing == "Consent" + + constraint.delete(db) + + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + async def test_publisher_override_defined_with_required_legal_basis_specified( + self, async_session_temp, db + ): + """Purpose override is defined, but no legal basis override""" + + resource = SystemSchema( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name=f"test_system_1_{uuid4()}", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="Collect data for content performance", + data_use="functional.service.improve", + legal_basis_for_processing="Consent", + data_categories=["user"], + ) + ], + ) + + system = await create_system( + resource, async_session_temp, CONFIG.security.oauth_root_client_id + ) + override = TCFPublisherOverride.create( + db, + data={ + "purpose": 10, + "is_included": True, + "required_legal_basis": "Legitimate interests", + }, + ) + pd = system.privacy_declarations[0] + + assert pd.purpose == 10 + assert await pd._publisher_override_legal_basis_join == "Legitimate interests" + assert await pd._publisher_override_is_included_join is True + assert await pd.overridden_legal_basis_for_processing == "Legitimate interests" + + override.delete(db) diff --git a/tests/ops/util/test_tcf_privacy_declarations.py b/tests/ops/util/test_tcf_privacy_declarations.py index 7b19dd7fb9..9d338265cf 100644 --- a/tests/ops/util/test_tcf_privacy_declarations.py +++ b/tests/ops/util/test_tcf_privacy_declarations.py @@ -1,163 +1,9 @@ -from uuid import uuid4 - import pytest -from fides.api.models.sql_models import PrivacyDeclaration from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride from fides.api.util.tcf.tcf_experience_contents import get_matching_privacy_declarations -class TestPrivacyDeclarationInstanceLevelHybridProperties: - """Test some instance-level hybrid properties defined on the Privacy Declaration - related to Publisher Overrides""" - - def test_privacy_declaration_enable_override_is_false(self, system, db): - """Enable override is false so overridden legal basis is going to default - to the defined legal basis""" - pd = PrivacyDeclaration( - name=f"declaration-name-{uuid4()}", - data_categories=[], - data_use="analytics.reporting.campaign_insights", - data_subjects=[], - data_qualifier="aggregated_data", - dataset_references=[], - ingress=None, - egress=None, - legal_basis_for_processing="Consent", - system_id=system.id, - ).save(db=db) - - assert pd.purpose == 9 - assert pd._publisher_override_legal_basis_join is None - assert pd._publisher_override_is_included_join is None - assert pd.overridden_legal_basis_for_processing == "Consent" - - @pytest.mark.usefixtures( - "enable_override_vendor_purposes", - ) - def test_enable_override_is_true_but_no_matching_purpose(self, system, db): - """Privacy Declaration has Special Purpose not Purpose, so no overrides applicable""" - pd = PrivacyDeclaration( - name=f"declaration-name-{uuid4()}", - data_categories=[], - data_use="essential.fraud_detection", - data_subjects=[], - data_qualifier="aggregated_data", - dataset_references=[], - ingress=None, - egress=None, - legal_basis_for_processing="Consent", - system_id=system.id, - ).save(db) - - assert pd.purpose is None - assert pd._publisher_override_legal_basis_join is None - assert pd._publisher_override_is_included_join is None - assert pd.overridden_legal_basis_for_processing == "Consent" - - @pytest.mark.usefixtures( - "enable_override_vendor_purposes", - ) - def test_enable_override_is_true_but_purpose_is_excluded(self, db, system): - """Purpose is overridden as excluded, so legal basis returns as None, to match - class-wide override""" - TCFPublisherOverride.create( - db, - data={ - "purpose": 9, - "is_included": False, - }, - ) - - pd = PrivacyDeclaration( - name="declaration-name", - data_categories=[], - data_use="analytics.reporting.campaign_insights", - data_subjects=[], - data_qualifier="aggregated_data", - dataset_references=[], - ingress=None, - egress=None, - flexible_legal_basis_for_processing=True, - legal_basis_for_processing="Consent", - system_id=system.id, - ).save(db=db) - - assert pd.purpose == 9 - assert pd._publisher_override_legal_basis_join is None - assert pd._publisher_override_is_included_join is False - assert pd.overridden_legal_basis_for_processing is None - - @pytest.mark.usefixtures( - "enable_override_vendor_purposes", - ) - def test_publisher_override_defined_but_no_required_legal_basis_specified( - self, db, system - ): - """Purpose override is defined, but no legal basis override""" - TCFPublisherOverride.create( - db, - data={ - "purpose": 9, - "is_included": True, - }, - ) - - pd = PrivacyDeclaration( - name="declaration-name", - data_categories=[], - data_use="analytics.reporting.campaign_insights", - data_subjects=[], - data_qualifier="aggregated_data", - dataset_references=[], - ingress=None, - egress=None, - flexible_legal_basis_for_processing=True, - legal_basis_for_processing="Consent", - system_id=system.id, - ).save(db=db) - - assert pd.purpose == 9 - assert pd._publisher_override_legal_basis_join is None - assert pd._publisher_override_is_included_join is True - assert pd.overridden_legal_basis_for_processing == "Consent" - - @pytest.mark.usefixtures( - "enable_override_vendor_purposes", - ) - def test_publisher_override_defined_with_required_legal_basis_specified( - self, db, system - ): - """Purpose override is defined, but no legal basis override""" - TCFPublisherOverride.create( - db, - data={ - "purpose": 9, - "is_included": True, - "required_legal_basis": "Legitimate interests", - }, - ) - - pd = PrivacyDeclaration( - name="declaration-name", - data_categories=[], - data_use="analytics.reporting.campaign_insights", - data_subjects=[], - data_qualifier="aggregated_data", - dataset_references=[], - ingress=None, - egress=None, - flexible_legal_basis_for_processing=True, - legal_basis_for_processing="Consent", - system_id=system.id, - ).save(db=db) - - assert pd.purpose == 9 - assert pd._publisher_override_legal_basis_join == "Legitimate interests" - assert pd._publisher_override_is_included_join is True - assert pd.overridden_legal_basis_for_processing == "Legitimate interests" - - class TestMatchingPrivacyDeclarations: """Tests matching privacy declarations returned that are the basis of the TCF Experience and the "relevant_systems" that are saved for consent reporting @@ -167,7 +13,7 @@ class TestMatchingPrivacyDeclarations: "emerse_system", ) def test_get_matching_privacy_declarations_enable_purpose_override_is_false( - self, emerse_system, db + self, db ): declarations = get_matching_privacy_declarations(db) From 64a14c2ceb214510b8e3fb44bc884e99a61e9c38 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 30 Nov 2023 12:31:55 -0600 Subject: [PATCH 07/15] Switch to creating a subquery to join the TCF Publisher overrides rather than relying on hybrid properties. - Remove all the hybrid properties from the PrivacyDeclaration model except for "purpose" which is still applicable - Rename/adapt a new method on the PrivacyDeclaration get_publisher_legal_basis_override() for use in supplementing the system response - For matching privacy declarations return override as "overridden_legal_basis_for_processing" --- src/fides/api/api/v1/endpoints/system.py | 6 +- src/fides/api/models/sql_models.py | 126 +++++------------- .../api/util/tcf/tcf_experience_contents.py | 70 ++++++++-- tests/ctl/core/test_api.py | 32 ++--- .../ops/util/test_tcf_privacy_declarations.py | 4 +- 5 files changed, 107 insertions(+), 131 deletions(-) diff --git a/src/fides/api/api/v1/endpoints/system.py b/src/fides/api/api/v1/endpoints/system.py index 329cd79f5b..f30098fbba 100644 --- a/src/fides/api/api/v1/endpoints/system.py +++ b/src/fides/api/api/v1/endpoints/system.py @@ -428,7 +428,9 @@ def instantiate_connection_from_template( return instantiate_connection(db, saas_connector_type, template_values, system) -async def supplement_privacy_declaration_response_with_legal_basis_override(resp: Union[Dict, System]) -> None: +async def supplement_privacy_declaration_response_with_legal_basis_override( + resp: Union[Dict, System] +) -> None: """At runtime, adds a "legal_basis_for_processing_override" to each PrivacyDeclaration""" for privacy_declaration in ( @@ -437,5 +439,5 @@ async def supplement_privacy_declaration_response_with_legal_basis_override(resp else resp.privacy_declarations ): privacy_declaration.legal_basis_for_processing_override = ( - await privacy_declaration.overridden_legal_basis_for_processing + await privacy_declaration.get_publisher_legal_basis_override() ) diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index 7c88198516..bb1851b4d1 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -7,7 +7,7 @@ from __future__ import annotations from enum import Enum as EnumType -from typing import Any, Dict, List, Optional, Set, Type, TypeVar, Union +from typing import Any, Dict, List, Optional, Set, Type, TypeVar from fideslang import MAPPED_PURPOSES_BY_DATA_USE from fideslang.gvl import MAPPED_PURPOSES, MappedPurpose @@ -30,15 +30,12 @@ type_coerce, ) from sqlalchemy.dialects.postgresql import ARRAY, BIGINT, BYTEA -from sqlalchemy.engine import Row from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession, async_object_session from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy.orm import Session, object_session, relationship -from sqlalchemy.orm.attributes import InstrumentedAttribute -from sqlalchemy.sql import func, Select +from sqlalchemy.orm import Session, relationship +from sqlalchemy.sql import Select, func from sqlalchemy.sql.elements import Case -from sqlalchemy.sql.selectable import ScalarSelect from sqlalchemy.sql.sqltypes import DateTime from typing_extensions import Protocol, runtime_checkable @@ -534,7 +531,7 @@ def create( @hybrid_property def purpose(self) -> Optional[int]: - """Returns the instance-level TCF Purpose.""" + """Returns the instance-level TCF Purpose if applicable.""" mapped_purpose: Optional[MappedPurpose] = MAPPED_PURPOSES_ONLY_BY_DATA_USE.get( self.data_use ) @@ -542,7 +539,7 @@ def purpose(self) -> Optional[int]: @purpose.expression def purpose(cls) -> Case: - """Returns the class-level TCF Purpose""" + """Returns the class-level TCF Purpose for use in a SQLAlchemy query""" return case( [ (cls.data_use == data_use, purpose.id) @@ -551,105 +548,52 @@ def purpose(cls) -> Case: else_=None, ) - @hybrid_property - async def _publisher_override_legal_basis_join(self) -> Optional[str]: - """Returns the instance-level overridden required legal basis""" - query: Select = select([TCFPublisherOverride.required_legal_basis]).where( - TCFPublisherOverride.purpose == self.purpose - ) - async_session: AsyncSession = async_object_session(self) - async with async_session.begin(): - result = await async_session.execute(query) - return result.scalars().first() - - @_publisher_override_legal_basis_join.expression - def _publisher_override_legal_basis_join(cls) -> ScalarSelect: - """Returns the class-level overridden required legal basis""" - return ( - select([TCFPublisherOverride.required_legal_basis]) - .where(TCFPublisherOverride.purpose == cls.purpose) - .as_scalar() - ) - - @hybrid_property - async def _publisher_override_is_included_join(self) -> Optional[bool]: - """Returns the instance-level indication of whether the purpose should be included""" - query: Select = select([TCFPublisherOverride.is_included]).where( - TCFPublisherOverride.purpose == self.purpose - ) - async_session: AsyncSession = async_object_session(self) - async with async_session.begin(): - result = await async_session.execute(query) - return result.scalars().first() - - @_publisher_override_is_included_join.expression - def _publisher_override_is_included_join(cls) -> ScalarSelect: - """Returns the class-level indication of whether the purpose should be included""" - return ( - select([TCFPublisherOverride.is_included]) - .where(TCFPublisherOverride.purpose == cls.purpose) - .as_scalar() - ) - - @hybrid_property - async def overridden_legal_basis_for_processing(self) -> Optional[str]: + async def get_publisher_legal_basis_override(self) -> Optional[str]: """ - Instance-level override of the legal basis for processing based on - publisher preferences. + Returns the overridden legal basis for processing based on TCF publisher overrides if applicable + + This is used for supplementing System responses with this override information. """ if not ( CONFIG.consent.override_vendor_purposes and self.flexible_legal_basis_for_processing ): + # Just return the default legal basis on this declaration if the override feature is disabled return self.legal_basis_for_processing - is_included: Optional[bool] = await self._publisher_override_is_included_join + query: Select = select( + [ + TCFPublisherOverride.is_included, + TCFPublisherOverride.required_legal_basis, + ] + ).where(TCFPublisherOverride.purpose == self.purpose) + + async_session: AsyncSession = async_object_session(self) + async with async_session.begin(): + result = await async_session.execute(query) + result = result.first() + + is_included: Optional[bool] = None + required_legal_basis: Optional[str] = None + + if result: + is_included = result.is_included + required_legal_basis = result.required_legal_basis if is_included is False: - # Overriding to False to match behavior of class-level override. - # Class-level override of legal basis to None removes Privacy Declaration - # from Experience + # If the Purpose is specified as excluded, just return None for the legal basis. + # This matches what we do when building the TCF Experience, as a null legal basis + # prevents the purpose from showing up in the TCF Experience altogether. return None - overridden_legal_basis: Optional[str] = ( - await self._publisher_override_legal_basis_join - ) - + # Now return the Fides-wide legal basis override for this purpose if it exists, + # otherwise defaulting to the legal basis on the Declaration return ( - overridden_legal_basis - if overridden_legal_basis # pylint: disable=using-constant-test + required_legal_basis + if required_legal_basis else self.legal_basis_for_processing ) - @overridden_legal_basis_for_processing.expression - def overridden_legal_basis_for_processing( - cls, - ) -> Union[InstrumentedAttribute, Case]: - """ - Class-level override of the legal basis for processing based on - publisher preferences. - """ - if not CONFIG.consent.override_vendor_purposes: - return cls.legal_basis_for_processing - - return case( - [ - ( - cls.flexible_legal_basis_for_processing.is_(False), - cls.legal_basis_for_processing, - ), - ( - cls._publisher_override_is_included_join.is_(False), - None, - ), - ( - cls._publisher_override_legal_basis_join.is_(None), - cls.legal_basis_for_processing, - ), - ], - else_=cls._publisher_override_legal_basis_join, - ) - class SystemModel(BaseModel): fides_key: str diff --git a/src/fides/api/util/tcf/tcf_experience_contents.py b/src/fides/api/util/tcf/tcf_experience_contents.py index 03d10845ec..61d2beb338 100644 --- a/src/fides/api/util/tcf/tcf_experience_contents.py +++ b/src/fides/api/util/tcf/tcf_experience_contents.py @@ -10,14 +10,16 @@ ) from fideslang.models import LegalBasisForProcessingEnum from fideslang.validation import FidesKey -from sqlalchemy import and_, not_, or_ +from sqlalchemy import and_, case, not_, or_ from sqlalchemy.orm import Query, Session +from sqlalchemy.sql import Alias from sqlalchemy.sql.elements import BinaryExpression, BooleanClauseList from fides.api.models.sql_models import ( # type:ignore[attr-defined] PrivacyDeclaration, System, ) +from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride from fides.api.schemas.base_class import FidesSchema from fides.api.schemas.tcf import ( TCFFeatureRecord, @@ -57,11 +59,10 @@ not_(System.vendor_id.startswith(AC_PREFIX)), System.vendor_id.is_(None) ) CONSENT_LEGAL_BASIS_FILTER: BinaryExpression = ( - PrivacyDeclaration.overridden_legal_basis_for_processing - == LegalBasisForProcessingEnum.CONSENT + PrivacyDeclaration.legal_basis_for_processing == LegalBasisForProcessingEnum.CONSENT ) LEGITIMATE_INTEREST_LEGAL_BASIS_FILTER: BinaryExpression = ( - PrivacyDeclaration.overridden_legal_basis_for_processing + PrivacyDeclaration.legal_basis_for_processing == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST ) @@ -150,6 +151,47 @@ class TCFExperienceContents( tcf_system_relationships: List[TCFVendorRelationships] = [] +def get_legal_basis_override_subquery(db: Session) -> Alias: + """Builds a subquery containing legal basis overrides for the Privacy Declaration if applicable""" + if not CONFIG.consent.override_vendor_purposes: + return db.query( + PrivacyDeclaration.id, + PrivacyDeclaration.legal_basis_for_processing.label( + "overridden_legal_basis_for_processing" + ), + ).subquery() + + return ( + db.query( + PrivacyDeclaration.id, + case( + [ + ( + PrivacyDeclaration.flexible_legal_basis_for_processing.is_( + False + ), + PrivacyDeclaration.legal_basis_for_processing, + ), + ( + TCFPublisherOverride.is_included.is_(False), + None, + ), + ( + TCFPublisherOverride.required_legal_basis.is_(None), + PrivacyDeclaration.legal_basis_for_processing, + ), + ], + else_=TCFPublisherOverride.required_legal_basis, + ).label("overridden_legal_basis_for_processing"), + ) + .outerjoin( + TCFPublisherOverride, + TCFPublisherOverride.purpose == PrivacyDeclaration.purpose, + ) + .subquery() + ) + + def get_matching_privacy_declarations(db: Session) -> Query: """Returns flattened system/privacy declaration records where we have a matching gvl data use AND the overridden legal basis for processing is "Consent" or "Legitimate interests". @@ -159,6 +201,8 @@ def get_matching_privacy_declarations(db: Session) -> Query: Only systems that meet this criteria should show up in the TCF overlay. """ + legal_basis_override_subquery = get_legal_basis_override_subquery(db) + matching_privacy_declarations: Query = ( db.query( System.id.label("system_id"), @@ -175,28 +219,29 @@ def get_matching_privacy_declarations(db: Session) -> Query: System.privacy_policy.label("system_privacy_policy"), System.vendor_id, PrivacyDeclaration.data_use, - PrivacyDeclaration.overridden_legal_basis_for_processing.label( # pylint: disable=no-member - "legal_basis_for_processing" + legal_basis_override_subquery.c.overridden_legal_basis_for_processing.label( # pylint: disable=no-member + "overridden_legal_basis_for_processing" ), PrivacyDeclaration.features, PrivacyDeclaration.retention_period, - PrivacyDeclaration.flexible_legal_basis_for_processing, PrivacyDeclaration.purpose, - PrivacyDeclaration.legal_basis_for_processing.label( - "original_legal_basis_for_processing" - ), + PrivacyDeclaration.legal_basis_for_processing, ) .outerjoin(PrivacyDeclaration, System.id == PrivacyDeclaration.system_id) + .outerjoin( + legal_basis_override_subquery, + legal_basis_override_subquery.c.id == PrivacyDeclaration.id, + ) .filter( or_( and_( GVL_DATA_USE_FILTER, - PrivacyDeclaration.overridden_legal_basis_for_processing + legal_basis_override_subquery.c.overridden_legal_basis_for_processing == LegalBasisForProcessingEnum.CONSENT, ), and_( GVL_DATA_USE_FILTER, - PrivacyDeclaration.overridden_legal_basis_for_processing + legal_basis_override_subquery.c.overridden_legal_basis_for_processing == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST, NOT_AC_SYSTEM_FILTER, ), @@ -212,6 +257,7 @@ def get_matching_privacy_declarations(db: Session) -> Query: matching_privacy_declarations = matching_privacy_declarations.filter( NOT_AC_SYSTEM_FILTER ) + return matching_privacy_declarations diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index bc9eadb35e..a2288234ee 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -683,8 +683,7 @@ async def test_system_create( assert privacy_decl.features == ["Link different devices"] assert privacy_decl.legal_basis_for_processing == "Public interest" assert ( - await privacy_decl.overridden_legal_basis_for_processing - == "Public interest" + await privacy_decl.get_publisher_legal_basis_override() == "Public interest" ) assert ( privacy_decl.impact_assessment_location @@ -1049,9 +1048,7 @@ async def test_get_overridden_declaration_legal_basis( None, ) assert decl.legal_basis_for_processing == "Consent" - assert ( - await decl.overridden_legal_basis_for_processing == "Legitimate interests" - ) + assert await decl.get_publisher_legal_basis_override() == "Legitimate interests" result = _api.get( url=test_config.cli.server_url, @@ -2670,10 +2667,7 @@ def test_trailing_slash(test_config: FidesConfig, endpoint_name: str) -> None: assert response.status_code == 200 -class TestPrivacyDeclarationInstanceLevelHybridProperties: - """Test some instance-level hybrid properties defined on the Privacy Declaration - related to Publisher Overrides""" - +class TestPrivacyDeclarationGetPublisherLegalBasisOverride: async def test_privacy_declaration_enable_override_is_false( self, async_session_temp ): @@ -2700,9 +2694,7 @@ async def test_privacy_declaration_enable_override_is_false( pd = system.privacy_declarations[0] assert pd.purpose == 9 - assert await pd._publisher_override_legal_basis_join is None - assert await pd._publisher_override_is_included_join is None - assert await pd.overridden_legal_basis_for_processing == "Consent" + assert await pd.get_publisher_legal_basis_override() == "Consent" @pytest.mark.usefixtures( "enable_override_vendor_purposes", @@ -2732,9 +2724,7 @@ async def test_enable_override_is_true_but_no_matching_purpose( pd = system.privacy_declarations[0] assert pd.purpose is None - assert await pd._publisher_override_legal_basis_join is None - assert await pd._publisher_override_is_included_join is None - assert await pd.overridden_legal_basis_for_processing == "Consent" + assert await pd.get_publisher_legal_basis_override() == "Consent" @pytest.mark.usefixtures( "enable_override_vendor_purposes", @@ -2773,9 +2763,7 @@ async def test_enable_override_is_true_but_purpose_is_excluded( ) assert pd.purpose == 5 - assert await pd._publisher_override_legal_basis_join is None - assert await pd._publisher_override_is_included_join is False - assert await pd.overridden_legal_basis_for_processing is None + assert await pd.get_publisher_legal_basis_override() is None constraint.delete(db) @@ -2815,9 +2803,7 @@ async def test_publisher_override_defined_but_no_required_legal_basis_specified( ) assert pd.purpose == 9 - assert await pd._publisher_override_legal_basis_join is None - assert await pd._publisher_override_is_included_join is True - assert await pd.overridden_legal_basis_for_processing == "Consent" + assert await pd.get_publisher_legal_basis_override() == "Consent" constraint.delete(db) @@ -2858,8 +2844,6 @@ async def test_publisher_override_defined_with_required_legal_basis_specified( pd = system.privacy_declarations[0] assert pd.purpose == 10 - assert await pd._publisher_override_legal_basis_join == "Legitimate interests" - assert await pd._publisher_override_is_included_join is True - assert await pd.overridden_legal_basis_for_processing == "Legitimate interests" + assert await pd.get_publisher_legal_basis_override() == "Legitimate interests" override.delete(db) diff --git a/tests/ops/util/test_tcf_privacy_declarations.py b/tests/ops/util/test_tcf_privacy_declarations.py index 9d338265cf..f7896b6a70 100644 --- a/tests/ops/util/test_tcf_privacy_declarations.py +++ b/tests/ops/util/test_tcf_privacy_declarations.py @@ -89,7 +89,7 @@ def test_privacy_declaration_publisher_overrides( declarations = get_matching_privacy_declarations(db) legal_basis_overrides = { - declaration.purpose: declaration.legal_basis_for_processing + declaration.purpose: declaration.overridden_legal_basis_for_processing for declaration in declarations if declaration.purpose } @@ -106,7 +106,7 @@ def test_privacy_declaration_publisher_overrides( } original_legal_basis = { - declaration.purpose: declaration.original_legal_basis_for_processing + declaration.purpose: declaration.legal_basis_for_processing for declaration in declarations if declaration.purpose } From 8bb1af35222a6eb8ea36a34f0983a3b05aec2202 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 30 Nov 2023 14:10:54 -0600 Subject: [PATCH 08/15] As the consent legal basis filter and legitimate interest legal basis filter cannot be defined up front anymore (they need access to the override legal basis subquery), return these filters from the matching privacy declarations method. This method has been renamed to "get_tcf_base_query_and_filters". --- .../api/util/tcf/tcf_experience_contents.py | 55 +++++++++++-------- tests/fixtures/application_fixtures.py | 15 +++++ tests/ops/models/test_privacy_preference.py | 41 ++++++++++++++ .../ops/util/test_tcf_privacy_declarations.py | 6 +- 4 files changed, 90 insertions(+), 27 deletions(-) diff --git a/src/fides/api/util/tcf/tcf_experience_contents.py b/src/fides/api/util/tcf/tcf_experience_contents.py index 61d2beb338..9b8d307d12 100644 --- a/src/fides/api/util/tcf/tcf_experience_contents.py +++ b/src/fides/api/util/tcf/tcf_experience_contents.py @@ -1,6 +1,6 @@ # mypy: disable-error-code="arg-type, attr-defined, assignment" from enum import Enum -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional, Tuple, Union from fideslang.gvl import ( MAPPED_PURPOSES, @@ -58,13 +58,6 @@ NOT_AC_SYSTEM_FILTER: BooleanClauseList = or_( not_(System.vendor_id.startswith(AC_PREFIX)), System.vendor_id.is_(None) ) -CONSENT_LEGAL_BASIS_FILTER: BinaryExpression = ( - PrivacyDeclaration.legal_basis_for_processing == LegalBasisForProcessingEnum.CONSENT -) -LEGITIMATE_INTEREST_LEGAL_BASIS_FILTER: BinaryExpression = ( - PrivacyDeclaration.legal_basis_for_processing - == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST -) GVL_DATA_USE_FILTER: BinaryExpression = PrivacyDeclaration.data_use.in_( ALL_GVL_DATA_USES @@ -192,17 +185,28 @@ def get_legal_basis_override_subquery(db: Session) -> Alias: ) -def get_matching_privacy_declarations(db: Session) -> Query: - """Returns flattened system/privacy declaration records where we have a matching gvl data use AND the - overridden legal basis for processing is "Consent" or "Legitimate interests". - - IMPORTANT - We are filtering against the "overridden_legal_basis_for_processing", not the defined "legal_basis_for_processing", - which takes into account potential Fides-wide publisher overrides. +def get_tcf_base_query_and_filters( + db: Session, +) -> Tuple[Query, BinaryExpression, BinaryExpression]: + """ + Returns the base query that contains the foundations of the TCF Experience as well as + two filters to further refine the query when building the Experience. - Only systems that meet this criteria should show up in the TCF overlay. + Rows show up corresponding to systems with GVL data uses and Legal bases of Consent or Legitimate interests. + AC systems are also included here. + Publisher overrides are applied at this stage which may suppress purposes or toggle the legal basis. """ legal_basis_override_subquery = get_legal_basis_override_subquery(db) + consent_legal_basis_filter: BinaryExpression = ( + legal_basis_override_subquery.c.overridden_legal_basis_for_processing + == LegalBasisForProcessingEnum.CONSENT + ) + legitimate_interest_legal_basis_filter: BinaryExpression = ( + legal_basis_override_subquery.c.overridden_legal_basis_for_processing + == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST + ) + matching_privacy_declarations: Query = ( db.query( System.id.label("system_id"), @@ -234,15 +238,10 @@ def get_matching_privacy_declarations(db: Session) -> Query: ) .filter( or_( + and_(GVL_DATA_USE_FILTER, consent_legal_basis_filter), and_( GVL_DATA_USE_FILTER, - legal_basis_override_subquery.c.overridden_legal_basis_for_processing - == LegalBasisForProcessingEnum.CONSENT, - ), - and_( - GVL_DATA_USE_FILTER, - legal_basis_override_subquery.c.overridden_legal_basis_for_processing - == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST, + legitimate_interest_legal_basis_filter, NOT_AC_SYSTEM_FILTER, ), AC_SYSTEM_NO_PRIVACY_DECL_FILTER, @@ -258,7 +257,11 @@ def get_matching_privacy_declarations(db: Session) -> Query: NOT_AC_SYSTEM_FILTER ) - return matching_privacy_declarations + return ( + matching_privacy_declarations, + consent_legal_basis_filter, + legitimate_interest_legal_basis_filter, + ) def systems_that_match_tcf_data_uses( @@ -295,7 +298,11 @@ def get_relevant_systems_for_tcf_attribute( # pylint: disable=too-many-return-s # For TCF attributes, we need to first filter to systems/privacy declarations that have a relevant GVL data use # as well as a legal basis of processing of consent or legitimate interests - starting_privacy_declarations: Query = get_matching_privacy_declarations(db) + ( + starting_privacy_declarations, + CONSENT_LEGAL_BASIS_FILTER, + LEGITIMATE_INTEREST_LEGAL_BASIS_FILTER, + ) = get_tcf_base_query_and_filters(db) purpose_data_uses: List[str] = [] if tcf_field in [ diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 252e4d51b3..cdd6f6ad92 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -68,6 +68,7 @@ _create_local_default_storage, default_storage_config_name, ) +from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride from fides.api.oauth.roles import APPROVER, VIEWER from fides.api.schemas.messaging.messaging import ( MessagingServiceDetails, @@ -3133,3 +3134,17 @@ def skimbit_system(db): }, ) return system + + +@pytest.fixture(scope="function") +def purpose_three_consent_publisher_override(db): + override = TCFPublisherOverride.create( + db, + data={ + "purpose": 3, + "is_included": True, + "required_legal_basis": "Consent", + }, + ) + yield override + override.delete(db) diff --git a/tests/ops/models/test_privacy_preference.py b/tests/ops/models/test_privacy_preference.py index 2a51ec7b04..46b1dcb1a9 100644 --- a/tests/ops/models/test_privacy_preference.py +++ b/tests/ops/models/test_privacy_preference.py @@ -1653,6 +1653,47 @@ def test_determine_relevant_systems_for_ac_system_purpose_legitimate_interests( == [] ) + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", "purpose_three_consent_publisher_override" + ) + def test_determine_relevant_systems_for_with_publisher_override( + self, + db, + system_with_no_uses, + ): + # Add data use to system that corresponds to purpose 3. Also has LI legal basis, but override sets it + # to Consent + pd_1 = PrivacyDeclaration.create( + db=db, + data={ + "name": "Collect data for content performance", + "system_id": system_with_no_uses.id, + "data_categories": ["user.device.cookie_id"], + "data_use": "marketing.advertising.profiling", + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "data_subjects": ["customer"], + "dataset_references": None, + "legal_basis_for_processing": "Legitimate interests", + "egress": None, + "ingress": None, + }, + ) + + assert PrivacyPreferenceHistory.determine_relevant_systems( + db, tcf_field=TCFComponentType.purpose_consent.value, tcf_value=3 + ) == [system_with_no_uses.fides_key] + + assert ( + PrivacyPreferenceHistory.determine_relevant_systems( + db, + tcf_field=TCFComponentType.purpose_legitimate_interests.value, + tcf_value=3, + ) + == [] + ) + + pd_1.delete(db) + class TestCurrentPrivacyPreference: def test_get_preference_by_notice_and_fides_user_device( diff --git a/tests/ops/util/test_tcf_privacy_declarations.py b/tests/ops/util/test_tcf_privacy_declarations.py index f7896b6a70..433011c652 100644 --- a/tests/ops/util/test_tcf_privacy_declarations.py +++ b/tests/ops/util/test_tcf_privacy_declarations.py @@ -1,7 +1,7 @@ import pytest from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride -from fides.api.util.tcf.tcf_experience_contents import get_matching_privacy_declarations +from fides.api.util.tcf.tcf_experience_contents import get_tcf_base_query_and_filters class TestMatchingPrivacyDeclarations: @@ -15,7 +15,7 @@ class TestMatchingPrivacyDeclarations: def test_get_matching_privacy_declarations_enable_purpose_override_is_false( self, db ): - declarations = get_matching_privacy_declarations(db) + declarations, _, _ = get_tcf_base_query_and_filters(db) assert declarations.count() == 13 @@ -86,7 +86,7 @@ def test_privacy_declaration_publisher_overrides( }, ) - declarations = get_matching_privacy_declarations(db) + declarations, _, _ = get_tcf_base_query_and_filters(db) legal_basis_overrides = { declaration.purpose: declaration.overridden_legal_basis_for_processing From 5b40058e815ec5e77db5299f8775fb084718724b Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 30 Nov 2023 14:45:35 -0600 Subject: [PATCH 09/15] Revert back to renaming the overridden legal basis as legal_basis_for_processing in the base TCF query --- src/fides/api/util/tcf/tcf_experience_contents.py | 4 ++-- tests/ops/util/test_tcf_privacy_declarations.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/fides/api/util/tcf/tcf_experience_contents.py b/src/fides/api/util/tcf/tcf_experience_contents.py index 9b8d307d12..436b25628c 100644 --- a/src/fides/api/util/tcf/tcf_experience_contents.py +++ b/src/fides/api/util/tcf/tcf_experience_contents.py @@ -224,12 +224,12 @@ def get_tcf_base_query_and_filters( System.vendor_id, PrivacyDeclaration.data_use, legal_basis_override_subquery.c.overridden_legal_basis_for_processing.label( # pylint: disable=no-member - "overridden_legal_basis_for_processing" + "legal_basis_for_processing" ), PrivacyDeclaration.features, PrivacyDeclaration.retention_period, PrivacyDeclaration.purpose, - PrivacyDeclaration.legal_basis_for_processing, + PrivacyDeclaration.legal_basis_for_processing.label("original_legal_basis_for_processing"), ) .outerjoin(PrivacyDeclaration, System.id == PrivacyDeclaration.system_id) .outerjoin( diff --git a/tests/ops/util/test_tcf_privacy_declarations.py b/tests/ops/util/test_tcf_privacy_declarations.py index 433011c652..6bd2cf619b 100644 --- a/tests/ops/util/test_tcf_privacy_declarations.py +++ b/tests/ops/util/test_tcf_privacy_declarations.py @@ -89,7 +89,7 @@ def test_privacy_declaration_publisher_overrides( declarations, _, _ = get_tcf_base_query_and_filters(db) legal_basis_overrides = { - declaration.purpose: declaration.overridden_legal_basis_for_processing + declaration.purpose: declaration.legal_basis_for_processing for declaration in declarations if declaration.purpose } @@ -106,7 +106,7 @@ def test_privacy_declaration_publisher_overrides( } original_legal_basis = { - declaration.purpose: declaration.legal_basis_for_processing + declaration.purpose: declaration.original_legal_basis_for_processing for declaration in declarations if declaration.purpose } From 9c97f2d094f019cdc0f59c735f8e2bb7738a57f0 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 30 Nov 2023 16:07:39 -0600 Subject: [PATCH 10/15] Rename table from TCFPublisherOverride to TCFPurposeOverride - Revert adding overrides to base system endpoints - this will go on another endpoint instead - Update database annotations --- .fides/db_dataset.yml | 21 +++++ ... => 5225ea4de265_tcf_purpose_overrides.py} | 14 +-- src/fides/api/api/v1/endpoints/system.py | 37 +------- src/fides/api/db/base.py | 2 +- src/fides/api/models/sql_models.py | 23 ++--- ..._overrides.py => tcf_purpose_overrides.py} | 6 +- src/fides/api/util/consent_util.py | 12 +-- .../api/util/tcf/tcf_experience_contents.py | 16 ++-- tests/ctl/core/test_api.py | 89 +++---------------- tests/fixtures/application_fixtures.py | 4 +- .../ops/util/test_tcf_privacy_declarations.py | 10 +-- 11 files changed, 82 insertions(+), 152 deletions(-) rename src/fides/api/alembic/migrations/versions/{5225ea4de265_tcf_publisher_overrides.py => 5225ea4de265_tcf_purpose_overrides.py} (76%) rename src/fides/api/models/{tcf_publisher_overrides.py => tcf_purpose_overrides.py} (84%) diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 324f6bacc2..5a6ac6886b 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -2797,5 +2797,26 @@ dataset: data_categories: [system.operations] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: updated_systems + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: tcf_purpose_overrides + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + fields: + - name: created_at + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: id + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: is_included + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: purpose + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: required_legal_basis + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: updated_at data_categories: [system.operations] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified \ No newline at end of file diff --git a/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py b/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_purpose_overrides.py similarity index 76% rename from src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py rename to src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_purpose_overrides.py index aa11c5edbf..f8791c9b23 100644 --- a/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_publisher_overrides.py +++ b/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_purpose_overrides.py @@ -1,4 +1,4 @@ -"""tcf_publisher_overrides +"""tcf_purpose_overrides Revision ID: 5225ea4de265 Revises: 1af6950f4625 @@ -17,7 +17,7 @@ def upgrade(): op.create_table( - "tcf_publisher_overrides", + "tcf_purpose_overrides", sa.Column("id", sa.String(length=255), nullable=False), sa.Column( "created_at", @@ -37,19 +37,19 @@ def upgrade(): sa.PrimaryKeyConstraint("id"), ) op.create_index( - op.f("ix_tcf_publisher_overrides_id"), - "tcf_publisher_overrides", + op.f("ix_tcf_purpose_overrides_id"), + "tcf_purpose_overrides", ["id"], unique=False, ) op.create_unique_constraint( - "purpose_constraint", "tcf_publisher_overrides", ["purpose"] + "purpose_constraint", "tcf_purpose_overrides", ["purpose"] ) def downgrade(): op.drop_index( - op.f("ix_tcf_publisher_overrides_id"), table_name="tcf_publisher_overrides" + op.f("ix_tcf_purpose_overrides_id"), table_name="tcf_purpose_overrides" ) - op.drop_table("tcf_publisher_overrides") + op.drop_table("tcf_purpose_overrides") diff --git a/src/fides/api/api/v1/endpoints/system.py b/src/fides/api/api/v1/endpoints/system.py index f30098fbba..59fa3be6a7 100644 --- a/src/fides/api/api/v1/endpoints/system.py +++ b/src/fides/api/api/v1/endpoints/system.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional from fastapi import Depends, HTTPException, Response, Security from fastapi_pagination import Page, Params @@ -258,10 +258,6 @@ async def update( updated_system, _ = await update_system( resource, db, current_user.id if current_user else None ) - await supplement_privacy_declaration_response_with_legal_basis_override( - updated_system - ) - return updated_system @@ -357,11 +353,7 @@ async def create( Override `System` create/POST to handle `.privacy_declarations` defined inline, for backward compatibility and ease of use for API users. """ - created = await create_system( - resource, db, current_user.id if current_user else None - ) - await supplement_privacy_declaration_response_with_legal_basis_override(created) - return created + return await create_system(resource, db, current_user.id if current_user else None) @SYSTEM_ROUTER.get( @@ -379,10 +371,7 @@ async def ls( # pylint: disable=invalid-name db: AsyncSession = Depends(get_async_db), ) -> List: """Get a list of all of the resources of this type.""" - systems = await list_resource(System, db) - for system in systems: - await supplement_privacy_declaration_response_with_legal_basis_override(system) - return systems + return await list_resource(System, db) @SYSTEM_ROUTER.get( @@ -400,10 +389,7 @@ async def get( db: AsyncSession = Depends(get_async_db), ) -> Dict: """Get a resource by its fides_key.""" - - resp = await get_resource_with_custom_fields(System, fides_key, db) - await supplement_privacy_declaration_response_with_legal_basis_override(resp) - return resp + return await get_resource_with_custom_fields(System, fides_key, db) @SYSTEM_CONNECTION_INSTANTIATE_ROUTER.post( @@ -426,18 +412,3 @@ def instantiate_connection_from_template( system = get_system(db, fides_key) return instantiate_connection(db, saas_connector_type, template_values, system) - - -async def supplement_privacy_declaration_response_with_legal_basis_override( - resp: Union[Dict, System] -) -> None: - """At runtime, adds a "legal_basis_for_processing_override" to each PrivacyDeclaration""" - - for privacy_declaration in ( - resp.get("privacy_declarations") - if isinstance(resp, Dict) - else resp.privacy_declarations - ): - privacy_declaration.legal_basis_for_processing_override = ( - await privacy_declaration.get_publisher_legal_basis_override() - ) diff --git a/src/fides/api/db/base.py b/src/fides/api/db/base.py index 1b29fe9fb3..9a6b37855c 100644 --- a/src/fides/api/db/base.py +++ b/src/fides/api/db/base.py @@ -39,4 +39,4 @@ from fides.api.models.system_compass_sync import SystemCompassSync from fides.api.models.system_history import SystemHistory from fides.api.models.system_manager import SystemManager -from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index bb1851b4d1..dfdc91b77b 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -45,7 +45,7 @@ from fides.api.models.client import ClientDetail from fides.api.models.fides_user import FidesUser from fides.api.models.fides_user_permissions import FidesUserPermissions -from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride from fides.config import get_config CONFIG = get_config() @@ -548,9 +548,9 @@ def purpose(cls) -> Case: else_=None, ) - async def get_publisher_legal_basis_override(self) -> Optional[str]: + async def get_purpose_legal_basis_override(self) -> Optional[str]: """ - Returns the overridden legal basis for processing based on TCF publisher overrides if applicable + Returns the overridden legal basis for processing based on TCF purpose overrides if applicable This is used for supplementing System responses with this override information. """ @@ -563,22 +563,23 @@ async def get_publisher_legal_basis_override(self) -> Optional[str]: query: Select = select( [ - TCFPublisherOverride.is_included, - TCFPublisherOverride.required_legal_basis, + TCFPurposeOverride.is_included, + TCFPurposeOverride.required_legal_basis, ] - ).where(TCFPublisherOverride.purpose == self.purpose) + ).where(TCFPurposeOverride.purpose == self.purpose) async_session: AsyncSession = async_object_session(self) + async with async_session.begin(): result = await async_session.execute(query) result = result.first() - is_included: Optional[bool] = None - required_legal_basis: Optional[str] = None + if not result: + # No purpose override saved + return self.legal_basis_for_processing - if result: - is_included = result.is_included - required_legal_basis = result.required_legal_basis + is_included: Optional[bool] = result.is_included + required_legal_basis: Optional[str] = result.required_legal_basis if is_included is False: # If the Purpose is specified as excluded, just return None for the legal basis. diff --git a/src/fides/api/models/tcf_publisher_overrides.py b/src/fides/api/models/tcf_purpose_overrides.py similarity index 84% rename from src/fides/api/models/tcf_publisher_overrides.py rename to src/fides/api/models/tcf_purpose_overrides.py index b326dd4ceb..4dad1398ba 100644 --- a/src/fides/api/models/tcf_publisher_overrides.py +++ b/src/fides/api/models/tcf_purpose_overrides.py @@ -4,9 +4,9 @@ from fides.api.db.base_class import Base -class TCFPublisherOverride(Base): +class TCFPurposeOverride(Base): """ - Stores TCF Publisher Overrides + Stores TCF Purpose Overrides Allows a customer to override Fides-wide which Purposes show up in the TCF Experience, and specify a global legal basis for that Purpose. @@ -14,7 +14,7 @@ class TCFPublisherOverride(Base): @declared_attr def __tablename__(self) -> str: - return "tcf_publisher_overrides" + return "tcf_purpose_overrides" purpose = Column(Integer, nullable=False) is_included = Column(Boolean, server_default="t", default=True) diff --git a/src/fides/api/util/consent_util.py b/src/fides/api/util/consent_util.py index 65ad4910bd..9741fb81c1 100644 --- a/src/fides/api/util/consent_util.py +++ b/src/fides/api/util/consent_util.py @@ -34,7 +34,7 @@ ProvidedIdentityType, ) from fides.api.models.sql_models import DataUse, System # type: ignore[attr-defined] -from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride from fides.api.schemas.privacy_experience import ExperienceConfigCreateWithId from fides.api.schemas.privacy_notice import PrivacyNoticeCreation, PrivacyNoticeWithId from fides.api.schemas.redis_cache import Identity @@ -676,19 +676,19 @@ def create_tcf_experiences_on_startup(db: Session) -> List[PrivacyExperience]: def create_default_tcf_publisher_overrides_on_startup( db: Session, -) -> List[TCFPublisherOverride]: +) -> List[TCFPurposeOverride]: """On startup, load default Publisher Overrides, one for each purpose, with a default of is_included=True and no legal basis override""" - publisher_overrides_created: List[TCFPublisherOverride] = [] + publisher_overrides_created: List[TCFPurposeOverride] = [] for purpose_id in range(1, 12): if ( - not db.query(TCFPublisherOverride) - .filter(TCFPublisherOverride.purpose == purpose_id) + not db.query(TCFPurposeOverride) + .filter(TCFPurposeOverride.purpose == purpose_id) .first() ): publisher_overrides_created.append( - TCFPublisherOverride.create( + TCFPurposeOverride.create( db, data={"purpose": purpose_id, "is_included": True} ) ) diff --git a/src/fides/api/util/tcf/tcf_experience_contents.py b/src/fides/api/util/tcf/tcf_experience_contents.py index 436b25628c..03da9dc7f4 100644 --- a/src/fides/api/util/tcf/tcf_experience_contents.py +++ b/src/fides/api/util/tcf/tcf_experience_contents.py @@ -19,7 +19,7 @@ PrivacyDeclaration, System, ) -from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride from fides.api.schemas.base_class import FidesSchema from fides.api.schemas.tcf import ( TCFFeatureRecord, @@ -166,20 +166,20 @@ def get_legal_basis_override_subquery(db: Session) -> Alias: PrivacyDeclaration.legal_basis_for_processing, ), ( - TCFPublisherOverride.is_included.is_(False), + TCFPurposeOverride.is_included.is_(False), None, ), ( - TCFPublisherOverride.required_legal_basis.is_(None), + TCFPurposeOverride.required_legal_basis.is_(None), PrivacyDeclaration.legal_basis_for_processing, ), ], - else_=TCFPublisherOverride.required_legal_basis, + else_=TCFPurposeOverride.required_legal_basis, ).label("overridden_legal_basis_for_processing"), ) .outerjoin( - TCFPublisherOverride, - TCFPublisherOverride.purpose == PrivacyDeclaration.purpose, + TCFPurposeOverride, + TCFPurposeOverride.purpose == PrivacyDeclaration.purpose, ) .subquery() ) @@ -229,7 +229,9 @@ def get_tcf_base_query_and_filters( PrivacyDeclaration.features, PrivacyDeclaration.retention_period, PrivacyDeclaration.purpose, - PrivacyDeclaration.legal_basis_for_processing.label("original_legal_basis_for_processing"), + PrivacyDeclaration.legal_basis_for_processing.label( + "original_legal_basis_for_processing" + ), ) .outerjoin(PrivacyDeclaration, System.id == PrivacyDeclaration.system_id) .outerjoin( diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index a2288234ee..c235045f4d 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -32,7 +32,7 @@ from fides.api.models.datasetconfig import DatasetConfig from fides.api.models.sql_models import Dataset, PrivacyDeclaration, System from fides.api.models.system_history import SystemHistory -from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride from fides.api.oauth.roles import OWNER, VIEWER from fides.api.schemas.system import PrivacyDeclarationResponse, SystemResponse from fides.api.schemas.taxonomy_extensions import ( @@ -683,7 +683,7 @@ async def test_system_create( assert privacy_decl.features == ["Link different devices"] assert privacy_decl.legal_basis_for_processing == "Public interest" assert ( - await privacy_decl.get_publisher_legal_basis_override() == "Public interest" + await privacy_decl.get_purpose_legal_basis_override() == "Public interest" ) assert ( privacy_decl.impact_assessment_location @@ -842,10 +842,7 @@ async def test_system_create_custom_metadata_saas_config( assert result.status_code == HTTP_201_CREATED assert result.json()["name"] == "Test System" assert len(result.json()["privacy_declarations"]) == 2 - assert { - decl["legal_basis_for_processing_override"] - for decl in result.json()["privacy_declarations"] - } == {"Public interest", None} + assert result.json()["meta"] == { "saas_config": { "type": "stripe", @@ -1007,68 +1004,6 @@ def test_data_stewards_included_in_response( assert "first_name" in steward assert "last_name" in steward - @pytest.mark.usefixtures("enable_override_vendor_purposes") - async def test_get_overridden_declaration_legal_basis( - self, db, test_config, async_session_temp - ): - resource = SystemSchema( - fides_key=str(uuid4()), - organization_fides_key="default_organization", - name=f"test_system_1_{uuid4()}", - system_type="test", - privacy_declarations=[ - PrivacyDeclarationSchema( - name="Collect data for content performance", - data_use="analytics.reporting.content_performance", - legal_basis_for_processing="Consent", - data_categories=["user"], - ) - ], - ) - - system = await create_system( - resource, async_session_temp, CONFIG.security.oauth_root_client_id - ) - - TCFPublisherOverride.create( - db, - data={ - "purpose": 8, - "is_included": True, - "required_legal_basis": "Legitimate interests", - }, - ) - - decl = next( - ( - declaration - for declaration in system.privacy_declarations - if declaration.data_use == "analytics.reporting.content_performance" - ), - None, - ) - assert decl.legal_basis_for_processing == "Consent" - assert await decl.get_publisher_legal_basis_override() == "Legitimate interests" - - result = _api.get( - url=test_config.cli.server_url, - headers=test_config.user.auth_header, - resource_type="system", - resource_id=system.fides_key, - ) - assert result.status_code == 200 - assert result.json()["fides_key"] == system.fides_key - assert ( - result.json()["privacy_declarations"][0]["legal_basis_for_processing"] - == "Consent" - ) - assert ( - result.json()["privacy_declarations"][0][ - "legal_basis_for_processing_override" - ] - == "Legitimate interests" - ) - @pytest.mark.unit class TestSystemUpdate: @@ -2667,7 +2602,7 @@ def test_trailing_slash(test_config: FidesConfig, endpoint_name: str) -> None: assert response.status_code == 200 -class TestPrivacyDeclarationGetPublisherLegalBasisOverride: +class TestPrivacyDeclarationGetPurposeLegalBasisOverride: async def test_privacy_declaration_enable_override_is_false( self, async_session_temp ): @@ -2694,7 +2629,7 @@ async def test_privacy_declaration_enable_override_is_false( pd = system.privacy_declarations[0] assert pd.purpose == 9 - assert await pd.get_publisher_legal_basis_override() == "Consent" + assert await pd.get_purpose_legal_basis_override() == "Consent" @pytest.mark.usefixtures( "enable_override_vendor_purposes", @@ -2724,7 +2659,7 @@ async def test_enable_override_is_true_but_no_matching_purpose( pd = system.privacy_declarations[0] assert pd.purpose is None - assert await pd.get_publisher_legal_basis_override() == "Consent" + assert await pd.get_purpose_legal_basis_override() == "Consent" @pytest.mark.usefixtures( "enable_override_vendor_purposes", @@ -2754,7 +2689,7 @@ async def test_enable_override_is_true_but_purpose_is_excluded( ) pd = system.privacy_declarations[0] - constraint = TCFPublisherOverride.create( + constraint = TCFPurposeOverride.create( db, data={ "purpose": 5, @@ -2763,7 +2698,7 @@ async def test_enable_override_is_true_but_purpose_is_excluded( ) assert pd.purpose == 5 - assert await pd.get_publisher_legal_basis_override() is None + assert await pd.get_purpose_legal_basis_override() is None constraint.delete(db) @@ -2794,7 +2729,7 @@ async def test_publisher_override_defined_but_no_required_legal_basis_specified( ) pd = system.privacy_declarations[0] - constraint = TCFPublisherOverride.create( + constraint = TCFPurposeOverride.create( db, data={ "purpose": 9, @@ -2803,7 +2738,7 @@ async def test_publisher_override_defined_but_no_required_legal_basis_specified( ) assert pd.purpose == 9 - assert await pd.get_publisher_legal_basis_override() == "Consent" + assert await pd.get_purpose_legal_basis_override() == "Consent" constraint.delete(db) @@ -2833,7 +2768,7 @@ async def test_publisher_override_defined_with_required_legal_basis_specified( system = await create_system( resource, async_session_temp, CONFIG.security.oauth_root_client_id ) - override = TCFPublisherOverride.create( + override = TCFPurposeOverride.create( db, data={ "purpose": 10, @@ -2844,6 +2779,6 @@ async def test_publisher_override_defined_with_required_legal_basis_specified( pd = system.privacy_declarations[0] assert pd.purpose == 10 - assert await pd.get_publisher_legal_basis_override() == "Legitimate interests" + assert await pd.get_purpose_legal_basis_override() == "Legitimate interests" override.delete(db) diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index cdd6f6ad92..9d193ca83f 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -68,7 +68,7 @@ _create_local_default_storage, default_storage_config_name, ) -from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride from fides.api.oauth.roles import APPROVER, VIEWER from fides.api.schemas.messaging.messaging import ( MessagingServiceDetails, @@ -3138,7 +3138,7 @@ def skimbit_system(db): @pytest.fixture(scope="function") def purpose_three_consent_publisher_override(db): - override = TCFPublisherOverride.create( + override = TCFPurposeOverride.create( db, data={ "purpose": 3, diff --git a/tests/ops/util/test_tcf_privacy_declarations.py b/tests/ops/util/test_tcf_privacy_declarations.py index 6bd2cf619b..cfe36b5b1b 100644 --- a/tests/ops/util/test_tcf_privacy_declarations.py +++ b/tests/ops/util/test_tcf_privacy_declarations.py @@ -1,6 +1,6 @@ import pytest -from fides.api.models.tcf_publisher_overrides import TCFPublisherOverride +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride from fides.api.util.tcf.tcf_experience_contents import get_tcf_base_query_and_filters @@ -49,14 +49,14 @@ def test_privacy_declaration_publisher_overrides( # Defined legal basis is also Consent for purpose 1 on Emerse. # Publisher override matches. - TCFPublisherOverride.create( + TCFPurposeOverride.create( db, data={"purpose": 1, "is_included": True, "required_legal_basis": "Consent"}, ) # Defined legal basis is Legitimate Interests for purpose 2 on Emerse. # Here, Purpose 2 is specified to be excluded. - TCFPublisherOverride.create( + TCFPurposeOverride.create( db, data={ "purpose": 2, @@ -66,7 +66,7 @@ def test_privacy_declaration_publisher_overrides( # Defined legal basis is Consent for purpose 3 on Emerse. # No legal basis override is defined. - TCFPublisherOverride.create( + TCFPurposeOverride.create( db, data={ "purpose": 3, @@ -77,7 +77,7 @@ def test_privacy_declaration_publisher_overrides( # Defined legal basis is Consent for purpose 4 on Emerse. # Override here has a different legal basis - TCFPublisherOverride.create( + TCFPurposeOverride.create( db, data={ "purpose": 4, From e6ea2dab76c0ef0f5edd8207f7237ae2829f214e Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 30 Nov 2023 17:03:01 -0600 Subject: [PATCH 11/15] Improve docstrings and test coverage. --- src/fides/api/app_setup.py | 14 ++--- src/fides/api/models/sql_models.py | 31 +++++++---- src/fides/api/schemas/system.py | 4 -- src/fides/api/util/consent_util.py | 10 +++- .../api/util/tcf/tcf_experience_contents.py | 14 ++++- tests/ops/util/test_consent_util.py | 13 +++++ .../ops/util/test_tcf_privacy_declarations.py | 55 +++++++++++++------ 7 files changed, 97 insertions(+), 44 deletions(-) diff --git a/src/fides/api/app_setup.py b/src/fides/api/app_setup.py index c9444a6880..3152e50932 100644 --- a/src/fides/api/app_setup.py +++ b/src/fides/api/app_setup.py @@ -41,7 +41,7 @@ from fides.api.service.saas_request.override_implementations import * from fides.api.util.cache import get_cache from fides.api.util.consent_util import ( - create_default_tcf_publisher_overrides_on_startup, + create_default_tcf_purpose_overrides_on_startup, create_tcf_experiences_on_startup, load_default_experience_configs_on_startup, load_default_notices_on_startup, @@ -207,7 +207,7 @@ async def run_database_startup(app: FastAPI) -> None: load_default_privacy_notices() # Similarly avoiding loading other consent out-of-the-box resources to avoid interfering with unit tests load_tcf_experiences() - load_tcf_publisher_overrides() + load_tcf_purpose_overrides() db.close() @@ -260,13 +260,13 @@ def load_tcf_experiences() -> None: db.close() -def load_tcf_publisher_overrides() -> None: - """Load default tcf publisher overrides""" - logger.info("Loading default TCF Publisher Overrides") +def load_tcf_purpose_overrides() -> None: + """Load default tcf purpose overrides""" + logger.info("Loading default TCF Purpose Overrides") try: db = get_api_session() - create_default_tcf_publisher_overrides_on_startup(db) + create_default_tcf_purpose_overrides_on_startup(db) except Exception as e: - logger.error("Skipping loading TCF Publisher Overrides: {}", str(e)) + logger.error("Skipping loading TCF Purpose Overrides: {}", str(e)) finally: db.close() diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index dfdc91b77b..e26a18b058 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -531,7 +531,11 @@ def create( @hybrid_property def purpose(self) -> Optional[int]: - """Returns the instance-level TCF Purpose if applicable.""" + """Returns the instance-level TCF Purpose if applicable. + + For example, if the data use on this Privacy Declaration is "marketing.advertising.profiling", + that corresponds to GVL Purpose 3, which would be returned here. + """ mapped_purpose: Optional[MappedPurpose] = MAPPED_PURPOSES_ONLY_BY_DATA_USE.get( self.data_use ) @@ -539,7 +543,11 @@ def purpose(self) -> Optional[int]: @purpose.expression def purpose(cls) -> Case: - """Returns the class-level TCF Purpose for use in a SQLAlchemy query""" + """Returns the class-level TCF Purpose for use in a SQLAlchemy query + + Since Purposes aren't stored directly on the Privacy Declaration, this comes in handy when + creating a query that joins on Purpose + """ return case( [ (cls.data_use == data_use, purpose.id) @@ -550,15 +558,22 @@ def purpose(cls) -> Case: async def get_purpose_legal_basis_override(self) -> Optional[str]: """ - Returns the overridden legal basis for processing based on TCF purpose overrides if applicable + Returns the legal basis for processing that factors in global purpose overrides if applicable. + + Original legal basis for processing is returned where: + - feature is disabled + - declaration's legal basis is not flexible + - no legal basis override specified + + Null is returned where: + - Purpose is excluded (this mimics what we do in the TCF Experience, which causes the purpose to be removed entirely) - This is used for supplementing System responses with this override information. + Otherwise, we return the override! """ if not ( CONFIG.consent.override_vendor_purposes and self.flexible_legal_basis_for_processing ): - # Just return the default legal basis on this declaration if the override feature is disabled return self.legal_basis_for_processing query: Select = select( @@ -575,20 +590,14 @@ async def get_purpose_legal_basis_override(self) -> Optional[str]: result = result.first() if not result: - # No purpose override saved return self.legal_basis_for_processing is_included: Optional[bool] = result.is_included required_legal_basis: Optional[str] = result.required_legal_basis if is_included is False: - # If the Purpose is specified as excluded, just return None for the legal basis. - # This matches what we do when building the TCF Experience, as a null legal basis - # prevents the purpose from showing up in the TCF Experience altogether. return None - # Now return the Fides-wide legal basis override for this purpose if it exists, - # otherwise defaulting to the legal basis on the Declaration return ( required_legal_basis if required_legal_basis diff --git a/src/fides/api/schemas/system.py b/src/fides/api/schemas/system.py index ddae949a37..278fc7ea76 100644 --- a/src/fides/api/schemas/system.py +++ b/src/fides/api/schemas/system.py @@ -19,10 +19,6 @@ class PrivacyDeclarationResponse(PrivacyDeclaration): ) cookies: Optional[List[Cookies]] = [] - legal_basis_for_processing_override: Optional[str] = Field( - description="Global overrides for this purpose's legal basis for processing if applicable. Defaults to the legal_basis_for_processing otherwise." - ) - class BasicSystemResponse(System): """ diff --git a/src/fides/api/util/consent_util.py b/src/fides/api/util/consent_util.py index 9741fb81c1..21ccaaf9dd 100644 --- a/src/fides/api/util/consent_util.py +++ b/src/fides/api/util/consent_util.py @@ -674,11 +674,15 @@ def create_tcf_experiences_on_startup(db: Session) -> List[PrivacyExperience]: return experiences_created -def create_default_tcf_publisher_overrides_on_startup( +def create_default_tcf_purpose_overrides_on_startup( db: Session, ) -> List[TCFPurposeOverride]: - """On startup, load default Publisher Overrides, one for each purpose, with a default of is_included=True - and no legal basis override""" + """On startup, load default Purpose Overrides, one for each purpose, with a default of is_included=True + and no legal basis override + + The defaults have no effect on what is returned in the TCF Privacy Experience, and this functionality needs + to be enabled via a config variable to be used at all. + """ publisher_overrides_created: List[TCFPurposeOverride] = [] for purpose_id in range(1, 12): diff --git a/src/fides/api/util/tcf/tcf_experience_contents.py b/src/fides/api/util/tcf/tcf_experience_contents.py index 03da9dc7f4..12bd2452d7 100644 --- a/src/fides/api/util/tcf/tcf_experience_contents.py +++ b/src/fides/api/util/tcf/tcf_experience_contents.py @@ -46,7 +46,6 @@ ALL_GVL_DATA_USES = list(set(PURPOSE_DATA_USES) | set(SPECIAL_PURPOSE_DATA_USES)) - # Common SQLAlchemy filters used below # Define a special-case filter for AC Systems with no Privacy Declarations @@ -145,7 +144,18 @@ class TCFExperienceContents( def get_legal_basis_override_subquery(db: Session) -> Alias: - """Builds a subquery containing legal basis overrides for the Privacy Declaration if applicable""" + """Subquery that allows us to globally override a purpose's legal basis for processing. + + Original legal basis for processing is returned where: + - feature is disabled + - declaration's legal basis is not flexible + - no legal basis override specified + + Null is returned where: + - Purpose is excluded (this will effectively remove the purpose from the Experience) + + Otherwise, we return the override! + """ if not CONFIG.consent.override_vendor_purposes: return db.query( PrivacyDeclaration.id, diff --git a/tests/ops/util/test_consent_util.py b/tests/ops/util/test_consent_util.py index 33d594506f..6c4cbd7184 100644 --- a/tests/ops/util/test_consent_util.py +++ b/tests/ops/util/test_consent_util.py @@ -30,6 +30,7 @@ add_errored_system_status_for_consent_reporting, cache_initial_status_and_identities_for_consent_reporting, create_default_experience_config, + create_default_tcf_purpose_overrides_on_startup, create_privacy_notices_util, create_tcf_experiences_on_startup, get_fides_user_device_id_provided_identity, @@ -1289,3 +1290,15 @@ def test_create_tcf_experiences_on_startup(self, db): experience_config = be_exp.experience_config assert experience_config.is_default assert experience_config.component == ComponentType.tcf_overlay + + +class TestLoadTCFPurposeOverrides: + def test_load_tcf_purpose_overrides_on_startup(self, db): + """Sanity check on creating TCF purpose overrides""" + default_override_objects_added = ( + create_default_tcf_purpose_overrides_on_startup(db) + ) + assert len(default_override_objects_added) == 11 + for override in default_override_objects_added: + assert override.is_included is True + assert override.required_legal_basis is None diff --git a/tests/ops/util/test_tcf_privacy_declarations.py b/tests/ops/util/test_tcf_privacy_declarations.py index cfe36b5b1b..bce95dd7e7 100644 --- a/tests/ops/util/test_tcf_privacy_declarations.py +++ b/tests/ops/util/test_tcf_privacy_declarations.py @@ -4,7 +4,7 @@ from fides.api.util.tcf.tcf_experience_contents import get_tcf_base_query_and_filters -class TestMatchingPrivacyDeclarations: +class TestBaseTCFQuery: """Tests matching privacy declarations returned that are the basis of the TCF Experience and the "relevant_systems" that are saved for consent reporting """ @@ -15,6 +15,7 @@ class TestMatchingPrivacyDeclarations: def test_get_matching_privacy_declarations_enable_purpose_override_is_false( self, db ): + """No legal bases are overridden because this features is off""" declarations, _, _ = get_tcf_base_query_and_filters(db) assert declarations.count() == 13 @@ -40,22 +41,32 @@ def test_get_matching_privacy_declarations_enable_purpose_override_is_false( "functional.storage": 1, } - @pytest.mark.usefixtures("emerse_system", "enable_override_vendor_purposes") - def test_privacy_declaration_publisher_overrides( - self, - db, - ): - """Define some purpose legal basis overrides and check their effects on what is returned in the Privacy Declaration query""" + @pytest.mark.usefixtures("enable_override_vendor_purposes") + def test_privacy_declaration_purpose_overrides(self, db, emerse_system): + """Comprehensive test of the various scenarios for privacy declaration purpose overrides when building the + base TCF query + + Some of the specifics aren't configurations that would be allowed for emerse, but ignore that here. + """ + + # As some test setup, override purpose 7 declaration to have an inflexible legal basis + purpose_7_decl = next( + decl + for decl in emerse_system.privacy_declarations + if decl.data_use == "analytics.reporting.ad_performance" + ) + purpose_7_decl.flexible_legal_basis_for_processing = False + purpose_7_decl.save(db) + + # For more test setup, create some purpose overrides - # Defined legal basis is also Consent for purpose 1 on Emerse. - # Publisher override matches. + # Purpose override that matches the legal basis already on the system's declaration TCFPurposeOverride.create( db, data={"purpose": 1, "is_included": True, "required_legal_basis": "Consent"}, ) - # Defined legal basis is Legitimate Interests for purpose 2 on Emerse. - # Here, Purpose 2 is specified to be excluded. + # Purpose override that marks Purpose 2 as excluded TCFPurposeOverride.create( db, data={ @@ -64,8 +75,7 @@ def test_privacy_declaration_publisher_overrides( }, ) - # Defined legal basis is Consent for purpose 3 on Emerse. - # No legal basis override is defined. + # Purpose override object defined, but no legal basis override specified TCFPurposeOverride.create( db, data={ @@ -75,8 +85,7 @@ def test_privacy_declaration_publisher_overrides( }, ) - # Defined legal basis is Consent for purpose 4 on Emerse. - # Override here has a different legal basis + # Purpose override defined with different legal basis than the one on the System's declaration TCFPurposeOverride.create( db, data={ @@ -86,6 +95,16 @@ def test_privacy_declaration_publisher_overrides( }, ) + # Purpose override defined with differing legal basis, however, purpose 7 above was marked as inflexible + TCFPurposeOverride.create( + db, + data={ + "purpose": 7, + "is_included": True, + "required_legal_basis": "Consent", + }, + ) + declarations, _, _ = get_tcf_base_query_and_filters(db) legal_basis_overrides = { @@ -94,8 +113,10 @@ def test_privacy_declaration_publisher_overrides( if declaration.purpose } - # Purpose 2 has been removed altogether and Purpose 4 Legal Basis - # has been overridden to Legitimate Interests legal basis + # Purpose 1 had no change, because the override matched + # Purpose 2 has been removed altogether + # Purpose 4 Legal Basis has been overridden to Legitimate Interests legal basis + # Purpose 7's override wasn't applied because that declaration was marked as inflexible. assert legal_basis_overrides == { 9: "Legitimate interests", 8: "Legitimate interests", From 09e6f1a05f280ecd1968b009c5b1faabff5d64c6 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 30 Nov 2023 17:04:03 -0600 Subject: [PATCH 12/15] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b4e9ee209..dcbbb2a57e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The types of changes are: - Added feature flag for separating system name and Compass vendor selector [#4437](https://github.com/ethyca/fides/pull/4437) - Fire GPP events per spec [#4433](https://github.com/ethyca/fides/pull/4433) - New override option `fides_tcf_gdpr_applies` for setting `gdprApplies` on the CMP API [#4453](https://github.com/ethyca/fides/pull/4453) +- Add support for global TCF Purpose Overrides [#4464](https://github.com/ethyca/fides/pull/4464) ### Changed - Improved bulk vendor adding table UX [#4425](https://github.com/ethyca/fides/pull/4425) From a3b0aca630b7eb8615c0098f03b79d2b920281bb Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 30 Nov 2023 17:25:52 -0600 Subject: [PATCH 13/15] Improve test coverage, remove unnecessary pylint override. --- src/fides/api/models/sql_models.py | 2 +- src/fides/api/util/consent_util.py | 6 +-- .../api/util/tcf/tcf_experience_contents.py | 2 +- tests/ctl/core/test_api.py | 49 +++++++++++++++++-- tests/ops/models/test_privacy_preference.py | 3 +- 5 files changed, 51 insertions(+), 11 deletions(-) diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index e26a18b058..1045b0f1a0 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -1,5 +1,5 @@ # type: ignore -# pylint: disable=comparison-with-callable,no-member +# pylint: disable=comparison-with-callable """ Contains all of the SqlAlchemy models for the Fides resources. """ diff --git a/src/fides/api/util/consent_util.py b/src/fides/api/util/consent_util.py index 21ccaaf9dd..ba287951e7 100644 --- a/src/fides/api/util/consent_util.py +++ b/src/fides/api/util/consent_util.py @@ -683,7 +683,7 @@ def create_default_tcf_purpose_overrides_on_startup( The defaults have no effect on what is returned in the TCF Privacy Experience, and this functionality needs to be enabled via a config variable to be used at all. """ - publisher_overrides_created: List[TCFPurposeOverride] = [] + purpose_override_resources_created: List[TCFPurposeOverride] = [] for purpose_id in range(1, 12): if ( @@ -691,10 +691,10 @@ def create_default_tcf_purpose_overrides_on_startup( .filter(TCFPurposeOverride.purpose == purpose_id) .first() ): - publisher_overrides_created.append( + purpose_override_resources_created.append( TCFPurposeOverride.create( db, data={"purpose": purpose_id, "is_included": True} ) ) - return publisher_overrides_created + return purpose_override_resources_created diff --git a/src/fides/api/util/tcf/tcf_experience_contents.py b/src/fides/api/util/tcf/tcf_experience_contents.py index 12bd2452d7..0ae55867e2 100644 --- a/src/fides/api/util/tcf/tcf_experience_contents.py +++ b/src/fides/api/util/tcf/tcf_experience_contents.py @@ -204,7 +204,7 @@ def get_tcf_base_query_and_filters( Rows show up corresponding to systems with GVL data uses and Legal bases of Consent or Legitimate interests. AC systems are also included here. - Publisher overrides are applied at this stage which may suppress purposes or toggle the legal basis. + Purpose overrides are applied at this stage which may suppress purposes or toggle the legal basis. """ legal_basis_override_subquery = get_legal_basis_override_subquery(db) diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index c235045f4d..8b4d987e5f 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -633,8 +633,8 @@ async def test_system_create( for i, decl in enumerate(system.privacy_declarations): for field in PrivacyDeclarationResponse.__fields__: - decl_val = getattr(decl, field, None) - if hasattr(decl, field) and isinstance(decl_val, typing.Hashable): + decl_val = getattr(decl, field) + if isinstance(decl_val, typing.Hashable): assert decl_val == json_results["privacy_declarations"][i][field] assert len(system.privacy_declarations) == 2 @@ -2702,13 +2702,53 @@ async def test_enable_override_is_true_but_purpose_is_excluded( constraint.delete(db) + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + async def test_legal_basis_is_inflexible(self, async_session_temp, db): + """Purpose is overridden but we can't apply because the legal basis is specified as inflexible""" + resource = SystemSchema( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name=f"test_system_1_{uuid4()}", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="Collect data for content performance", + data_use="personalize.content.profiling", + legal_basis_for_processing="Consent", + flexible_legal_basis_for_processing=False, + data_categories=["user"], + ) + ], + ) + + system = await create_system( + resource, async_session_temp, CONFIG.security.oauth_root_client_id + ) + pd = system.privacy_declarations[0] + + constraint = TCFPurposeOverride.create( + db, + data={ + "purpose": 5, + "is_included": True, + "required_legal_basis": "Legitimate interests", + }, + ) + + assert pd.purpose == 5 + assert await pd.get_purpose_legal_basis_override() == "Consent" + + constraint.delete(db) + @pytest.mark.usefixtures( "enable_override_vendor_purposes", ) async def test_publisher_override_defined_but_no_required_legal_basis_specified( self, db, async_session_temp ): - """Purpose override is defined, but no legal basis override""" + """Purpose override *object* is defined, but no legal basis override""" resource = SystemSchema( fides_key=str(uuid4()), organization_fides_key="default_organization", @@ -2748,8 +2788,7 @@ async def test_publisher_override_defined_but_no_required_legal_basis_specified( async def test_publisher_override_defined_with_required_legal_basis_specified( self, async_session_temp, db ): - """Purpose override is defined, but no legal basis override""" - + """Purpose override specified along with the requirements to apply that override""" resource = SystemSchema( fides_key=str(uuid4()), organization_fides_key="default_organization", diff --git a/tests/ops/models/test_privacy_preference.py b/tests/ops/models/test_privacy_preference.py index 46b1dcb1a9..2751e244f5 100644 --- a/tests/ops/models/test_privacy_preference.py +++ b/tests/ops/models/test_privacy_preference.py @@ -1656,11 +1656,12 @@ def test_determine_relevant_systems_for_ac_system_purpose_legitimate_interests( @pytest.mark.usefixtures( "enable_override_vendor_purposes", "purpose_three_consent_publisher_override" ) - def test_determine_relevant_systems_for_with_publisher_override( + def test_determine_relevant_systems_for_with_purpose_override( self, db, system_with_no_uses, ): + """Relevant system calculation takes into account legal basis overrides""" # Add data use to system that corresponds to purpose 3. Also has LI legal basis, but override sets it # to Consent pd_1 = PrivacyDeclaration.create( From 16892f0d8c1cf9fe1828bd7854b369ffcb427333 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sat, 2 Dec 2023 07:37:19 -0600 Subject: [PATCH 14/15] Index purpose --- ...py => 848a8f4125cf_tcf_purpose_overrides.py} | 17 +++++++++++++---- src/fides/api/models/tcf_purpose_overrides.py | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) rename src/fides/api/alembic/migrations/versions/{5225ea4de265_tcf_purpose_overrides.py => 848a8f4125cf_tcf_purpose_overrides.py} (79%) diff --git a/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_purpose_overrides.py b/src/fides/api/alembic/migrations/versions/848a8f4125cf_tcf_purpose_overrides.py similarity index 79% rename from src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_purpose_overrides.py rename to src/fides/api/alembic/migrations/versions/848a8f4125cf_tcf_purpose_overrides.py index f8791c9b23..628746b8ed 100644 --- a/src/fides/api/alembic/migrations/versions/5225ea4de265_tcf_purpose_overrides.py +++ b/src/fides/api/alembic/migrations/versions/848a8f4125cf_tcf_purpose_overrides.py @@ -1,15 +1,15 @@ """tcf_purpose_overrides -Revision ID: 5225ea4de265 +Revision ID: 848a8f4125cf Revises: 1af6950f4625 -Create Date: 2023-11-27 15:35:07.679747 +Create Date: 2023-12-02 13:18:55.768697 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. -revision = "5225ea4de265" +revision = "848a8f4125cf" down_revision = "1af6950f4625" branch_labels = None depends_on = None @@ -42,13 +42,22 @@ def upgrade(): ["id"], unique=False, ) - op.create_unique_constraint( "purpose_constraint", "tcf_purpose_overrides", ["purpose"] ) + op.create_index( + op.f("ix_tcf_purpose_overrides_purpose"), + "tcf_purpose_overrides", + ["purpose"], + unique=False, + ) + def downgrade(): + op.drop_index( + op.f("ix_tcf_purpose_overrides_purpose"), table_name="tcf_purpose_overrides" + ) op.drop_index( op.f("ix_tcf_purpose_overrides_id"), table_name="tcf_purpose_overrides" ) diff --git a/src/fides/api/models/tcf_purpose_overrides.py b/src/fides/api/models/tcf_purpose_overrides.py index 4dad1398ba..30943642e8 100644 --- a/src/fides/api/models/tcf_purpose_overrides.py +++ b/src/fides/api/models/tcf_purpose_overrides.py @@ -16,7 +16,7 @@ class TCFPurposeOverride(Base): def __tablename__(self) -> str: return "tcf_purpose_overrides" - purpose = Column(Integer, nullable=False) + purpose = Column(Integer, nullable=False, index=True) is_included = Column(Boolean, server_default="t", default=True) required_legal_basis = Column(String) From 563d52db435d3898e87d001ff0390a1b540e7fa8 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Sat, 2 Dec 2023 07:42:07 -0600 Subject: [PATCH 15/15] Bump downrev. --- .../migrations/versions/848a8f4125cf_tcf_purpose_overrides.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/848a8f4125cf_tcf_purpose_overrides.py b/src/fides/api/alembic/migrations/versions/848a8f4125cf_tcf_purpose_overrides.py index 628746b8ed..bae78a146a 100644 --- a/src/fides/api/alembic/migrations/versions/848a8f4125cf_tcf_purpose_overrides.py +++ b/src/fides/api/alembic/migrations/versions/848a8f4125cf_tcf_purpose_overrides.py @@ -1,7 +1,7 @@ """tcf_purpose_overrides Revision ID: 848a8f4125cf -Revises: 1af6950f4625 +Revises: 7f7c2b098f5d Create Date: 2023-12-02 13:18:55.768697 """ @@ -10,7 +10,7 @@ # revision identifiers, used by Alembic. revision = "848a8f4125cf" -down_revision = "1af6950f4625" +down_revision = "7f7c2b098f5d" branch_labels = None depends_on = None