diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 324f6bacc2..5a6ac6886b 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -2797,5 +2797,26 @@ dataset: data_categories: [system.operations] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: updated_systems + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: tcf_purpose_overrides + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + fields: + - name: created_at + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: id + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: is_included + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: purpose + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: required_legal_basis + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: updated_at data_categories: [system.operations] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified \ No newline at end of file diff --git a/.fides/fides.toml b/.fides/fides.toml index cc7fe748e0..f336635430 100644 --- a/.fides/fides.toml +++ b/.fides/fides.toml @@ -65,3 +65,4 @@ notification_service_type = "mailgun" [consent] tcf_enabled = false ac_enabled = false +override_vendor_purposes = false diff --git a/CHANGELOG.md b/CHANGELOG.md index d82077d62f..3a55fda9c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The types of changes are: - Dynamic importing for GPP bundle [#4447](https://github.com/ethyca/fides/pull/4447) - Paging to vendors in the TCF overlay [#4463](https://github.com/ethyca/fides/pull/4463) - New purposes endpoint and indices to improve system lookups [#4452](https://github.com/ethyca/fides/pull/4452) +- Add support for global TCF Purpose Overrides [#4464](https://github.com/ethyca/fides/pull/4464) ### Fixed - Fix type errors when TCF vendors have no dataDeclaration [#4465](https://github.com/ethyca/fides/pull/4465) diff --git a/src/fides/api/alembic/migrations/versions/848a8f4125cf_tcf_purpose_overrides.py b/src/fides/api/alembic/migrations/versions/848a8f4125cf_tcf_purpose_overrides.py new file mode 100644 index 0000000000..bae78a146a --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/848a8f4125cf_tcf_purpose_overrides.py @@ -0,0 +1,64 @@ +"""tcf_purpose_overrides + +Revision ID: 848a8f4125cf +Revises: 7f7c2b098f5d +Create Date: 2023-12-02 13:18:55.768697 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "848a8f4125cf" +down_revision = "7f7c2b098f5d" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "tcf_purpose_overrides", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("purpose", sa.Integer(), nullable=False), + sa.Column("is_included", sa.Boolean(), server_default="t", nullable=True), + sa.Column("required_legal_basis", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_tcf_purpose_overrides_id"), + "tcf_purpose_overrides", + ["id"], + unique=False, + ) + op.create_unique_constraint( + "purpose_constraint", "tcf_purpose_overrides", ["purpose"] + ) + + op.create_index( + op.f("ix_tcf_purpose_overrides_purpose"), + "tcf_purpose_overrides", + ["purpose"], + unique=False, + ) + + +def downgrade(): + op.drop_index( + op.f("ix_tcf_purpose_overrides_purpose"), table_name="tcf_purpose_overrides" + ) + op.drop_index( + op.f("ix_tcf_purpose_overrides_id"), table_name="tcf_purpose_overrides" + ) + op.drop_table("tcf_purpose_overrides") diff --git a/src/fides/api/app_setup.py b/src/fides/api/app_setup.py index 51f22c989f..3152e50932 100644 --- a/src/fides/api/app_setup.py +++ b/src/fides/api/app_setup.py @@ -41,6 +41,7 @@ from fides.api.service.saas_request.override_implementations import * from fides.api.util.cache import get_cache from fides.api.util.consent_util import ( + create_default_tcf_purpose_overrides_on_startup, create_tcf_experiences_on_startup, load_default_experience_configs_on_startup, load_default_notices_on_startup, @@ -204,7 +205,9 @@ async def run_database_startup(app: FastAPI) -> None: # Default notices subject to change, so preventing these from # loading in test mode to avoid interfering with unit tests. load_default_privacy_notices() + # Similarly avoiding loading other consent out-of-the-box resources to avoid interfering with unit tests load_tcf_experiences() + load_tcf_purpose_overrides() db.close() @@ -255,3 +258,15 @@ def load_tcf_experiences() -> None: logger.error("Skipping loading TCF Overlay Experiences: {}", str(e)) finally: db.close() + + +def load_tcf_purpose_overrides() -> None: + """Load default tcf purpose overrides""" + logger.info("Loading default TCF Purpose Overrides") + try: + db = get_api_session() + create_default_tcf_purpose_overrides_on_startup(db) + except Exception as e: + logger.error("Skipping loading TCF Purpose Overrides: {}", str(e)) + finally: + db.close() diff --git a/src/fides/api/db/base.py b/src/fides/api/db/base.py index 92caa47ea2..9a6b37855c 100644 --- a/src/fides/api/db/base.py +++ b/src/fides/api/db/base.py @@ -39,3 +39,4 @@ from fides.api.models.system_compass_sync import SystemCompassSync from fides.api.models.system_history import SystemHistory from fides.api.models.system_manager import SystemManager +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index 636039e9d5..1045b0f1a0 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -1,5 +1,5 @@ # type: ignore - +# pylint: disable=comparison-with-callable """ Contains all of the SqlAlchemy models for the Fides resources. """ @@ -9,6 +9,8 @@ from enum import Enum as EnumType from typing import Any, Dict, List, Optional, Set, Type, TypeVar +from fideslang import MAPPED_PURPOSES_BY_DATA_USE +from fideslang.gvl import MAPPED_PURPOSES, MappedPurpose from fideslang.models import DataCategory as FideslangDataCategory from fideslang.models import Dataset as FideslangDataset from pydantic import BaseModel @@ -22,13 +24,18 @@ Text, TypeDecorator, UniqueConstraint, + case, cast, + select, type_coerce, ) from sqlalchemy.dialects.postgresql import ARRAY, BIGINT, BYTEA from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession, async_object_session +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import Session, relationship -from sqlalchemy.sql import func +from sqlalchemy.sql import Select, func +from sqlalchemy.sql.elements import Case from sqlalchemy.sql.sqltypes import DateTime from typing_extensions import Protocol, runtime_checkable @@ -38,10 +45,18 @@ from fides.api.models.client import ClientDetail from fides.api.models.fides_user import FidesUser from fides.api.models.fides_user_permissions import FidesUserPermissions +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride from fides.config import get_config CONFIG = get_config() +# Mapping of data uses to *Purposes* not Special Purposes +MAPPED_PURPOSES_ONLY_BY_DATA_USE: Dict[str, MappedPurpose] = { + data_use: purpose + for data_use, purpose in MAPPED_PURPOSES_BY_DATA_USE.items() + if purpose in MAPPED_PURPOSES.values() +} + class FidesBase(FideslibBase): """ @@ -514,6 +529,81 @@ def create( """Overrides base create to avoid unique check on `name` column""" return super().create(db=db, data=data, check_name=check_name) + @hybrid_property + def purpose(self) -> Optional[int]: + """Returns the instance-level TCF Purpose if applicable. + + For example, if the data use on this Privacy Declaration is "marketing.advertising.profiling", + that corresponds to GVL Purpose 3, which would be returned here. + """ + mapped_purpose: Optional[MappedPurpose] = MAPPED_PURPOSES_ONLY_BY_DATA_USE.get( + self.data_use + ) + return mapped_purpose.id if mapped_purpose else None + + @purpose.expression + def purpose(cls) -> Case: + """Returns the class-level TCF Purpose for use in a SQLAlchemy query + + Since Purposes aren't stored directly on the Privacy Declaration, this comes in handy when + creating a query that joins on Purpose + """ + return case( + [ + (cls.data_use == data_use, purpose.id) + for data_use, purpose in MAPPED_PURPOSES_ONLY_BY_DATA_USE.items() + ], + else_=None, + ) + + async def get_purpose_legal_basis_override(self) -> Optional[str]: + """ + Returns the legal basis for processing that factors in global purpose overrides if applicable. + + Original legal basis for processing is returned where: + - feature is disabled + - declaration's legal basis is not flexible + - no legal basis override specified + + Null is returned where: + - Purpose is excluded (this mimics what we do in the TCF Experience, which causes the purpose to be removed entirely) + + Otherwise, we return the override! + """ + if not ( + CONFIG.consent.override_vendor_purposes + and self.flexible_legal_basis_for_processing + ): + return self.legal_basis_for_processing + + query: Select = select( + [ + TCFPurposeOverride.is_included, + TCFPurposeOverride.required_legal_basis, + ] + ).where(TCFPurposeOverride.purpose == self.purpose) + + async_session: AsyncSession = async_object_session(self) + + async with async_session.begin(): + result = await async_session.execute(query) + result = result.first() + + if not result: + return self.legal_basis_for_processing + + is_included: Optional[bool] = result.is_included + required_legal_basis: Optional[str] = result.required_legal_basis + + if is_included is False: + return None + + return ( + required_legal_basis + if required_legal_basis + else self.legal_basis_for_processing + ) + class SystemModel(BaseModel): fides_key: str diff --git a/src/fides/api/models/tcf_purpose_overrides.py b/src/fides/api/models/tcf_purpose_overrides.py new file mode 100644 index 0000000000..30943642e8 --- /dev/null +++ b/src/fides/api/models/tcf_purpose_overrides.py @@ -0,0 +1,23 @@ +from sqlalchemy import Boolean, Column, Integer, String, UniqueConstraint +from sqlalchemy.ext.declarative import declared_attr + +from fides.api.db.base_class import Base + + +class TCFPurposeOverride(Base): + """ + Stores TCF Purpose Overrides + + Allows a customer to override Fides-wide which Purposes show up in the TCF Experience, and + specify a global legal basis for that Purpose. + """ + + @declared_attr + def __tablename__(self) -> str: + return "tcf_purpose_overrides" + + purpose = Column(Integer, nullable=False, index=True) + is_included = Column(Boolean, server_default="t", default=True) + required_legal_basis = Column(String) + + UniqueConstraint("purpose") diff --git a/src/fides/api/util/consent_util.py b/src/fides/api/util/consent_util.py index bc0333fe42..ba287951e7 100644 --- a/src/fides/api/util/consent_util.py +++ b/src/fides/api/util/consent_util.py @@ -34,6 +34,7 @@ ProvidedIdentityType, ) from fides.api.models.sql_models import DataUse, System # type: ignore[attr-defined] +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride from fides.api.schemas.privacy_experience import ExperienceConfigCreateWithId from fides.api.schemas.privacy_notice import PrivacyNoticeCreation, PrivacyNoticeWithId from fides.api.schemas.redis_cache import Identity @@ -671,3 +672,29 @@ def create_tcf_experiences_on_startup(db: Session) -> List[PrivacyExperience]: ) return experiences_created + + +def create_default_tcf_purpose_overrides_on_startup( + db: Session, +) -> List[TCFPurposeOverride]: + """On startup, load default Purpose Overrides, one for each purpose, with a default of is_included=True + and no legal basis override + + The defaults have no effect on what is returned in the TCF Privacy Experience, and this functionality needs + to be enabled via a config variable to be used at all. + """ + purpose_override_resources_created: List[TCFPurposeOverride] = [] + + for purpose_id in range(1, 12): + if ( + not db.query(TCFPurposeOverride) + .filter(TCFPurposeOverride.purpose == purpose_id) + .first() + ): + purpose_override_resources_created.append( + TCFPurposeOverride.create( + db, data={"purpose": purpose_id, "is_included": True} + ) + ) + + return purpose_override_resources_created diff --git a/src/fides/api/util/tcf/tcf_experience_contents.py b/src/fides/api/util/tcf/tcf_experience_contents.py index 88f94d88fb..0ae55867e2 100644 --- a/src/fides/api/util/tcf/tcf_experience_contents.py +++ b/src/fides/api/util/tcf/tcf_experience_contents.py @@ -1,6 +1,6 @@ # mypy: disable-error-code="arg-type, attr-defined, assignment" from enum import Enum -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional, Tuple, Union from fideslang.gvl import ( MAPPED_PURPOSES, @@ -10,14 +10,16 @@ ) from fideslang.models import LegalBasisForProcessingEnum from fideslang.validation import FidesKey -from sqlalchemy import and_, not_, or_ +from sqlalchemy import and_, case, not_, or_ from sqlalchemy.orm import Query, Session +from sqlalchemy.sql import Alias from sqlalchemy.sql.elements import BinaryExpression, BooleanClauseList from fides.api.models.sql_models import ( # type:ignore[attr-defined] PrivacyDeclaration, System, ) +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride from fides.api.schemas.base_class import FidesSchema from fides.api.schemas.tcf import ( TCFFeatureRecord, @@ -44,7 +46,6 @@ ALL_GVL_DATA_USES = list(set(PURPOSE_DATA_USES) | set(SPECIAL_PURPOSE_DATA_USES)) - # Common SQLAlchemy filters used below # Define a special-case filter for AC Systems with no Privacy Declarations @@ -56,13 +57,6 @@ NOT_AC_SYSTEM_FILTER: BooleanClauseList = or_( not_(System.vendor_id.startswith(AC_PREFIX)), System.vendor_id.is_(None) ) -CONSENT_LEGAL_BASIS_FILTER: BinaryExpression = ( - PrivacyDeclaration.legal_basis_for_processing == LegalBasisForProcessingEnum.CONSENT -) -LEGITIMATE_INTEREST_LEGAL_BASIS_FILTER: BinaryExpression = ( - PrivacyDeclaration.legal_basis_for_processing - == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST -) GVL_DATA_USE_FILTER: BinaryExpression = PrivacyDeclaration.data_use.in_( ALL_GVL_DATA_USES @@ -149,12 +143,80 @@ class TCFExperienceContents( tcf_system_relationships: List[TCFVendorRelationships] = [] -def get_matching_privacy_declarations(db: Session) -> Query: - """Returns flattened system/privacy declaration records where we have a matching gvl data use AND the - legal basis for processing is "Consent" or "Legitimate interests" +def get_legal_basis_override_subquery(db: Session) -> Alias: + """Subquery that allows us to globally override a purpose's legal basis for processing. + + Original legal basis for processing is returned where: + - feature is disabled + - declaration's legal basis is not flexible + - no legal basis override specified + + Null is returned where: + - Purpose is excluded (this will effectively remove the purpose from the Experience) + + Otherwise, we return the override! + """ + if not CONFIG.consent.override_vendor_purposes: + return db.query( + PrivacyDeclaration.id, + PrivacyDeclaration.legal_basis_for_processing.label( + "overridden_legal_basis_for_processing" + ), + ).subquery() - Only systems that meet this criteria should show up in the TCF overlay. + return ( + db.query( + PrivacyDeclaration.id, + case( + [ + ( + PrivacyDeclaration.flexible_legal_basis_for_processing.is_( + False + ), + PrivacyDeclaration.legal_basis_for_processing, + ), + ( + TCFPurposeOverride.is_included.is_(False), + None, + ), + ( + TCFPurposeOverride.required_legal_basis.is_(None), + PrivacyDeclaration.legal_basis_for_processing, + ), + ], + else_=TCFPurposeOverride.required_legal_basis, + ).label("overridden_legal_basis_for_processing"), + ) + .outerjoin( + TCFPurposeOverride, + TCFPurposeOverride.purpose == PrivacyDeclaration.purpose, + ) + .subquery() + ) + + +def get_tcf_base_query_and_filters( + db: Session, +) -> Tuple[Query, BinaryExpression, BinaryExpression]: """ + Returns the base query that contains the foundations of the TCF Experience as well as + two filters to further refine the query when building the Experience. + + Rows show up corresponding to systems with GVL data uses and Legal bases of Consent or Legitimate interests. + AC systems are also included here. + Purpose overrides are applied at this stage which may suppress purposes or toggle the legal basis. + """ + legal_basis_override_subquery = get_legal_basis_override_subquery(db) + + consent_legal_basis_filter: BinaryExpression = ( + legal_basis_override_subquery.c.overridden_legal_basis_for_processing + == LegalBasisForProcessingEnum.CONSENT + ) + legitimate_interest_legal_basis_filter: BinaryExpression = ( + legal_basis_override_subquery.c.overridden_legal_basis_for_processing + == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST + ) + matching_privacy_declarations: Query = ( db.query( System.id.label("system_id"), @@ -171,22 +233,27 @@ def get_matching_privacy_declarations(db: Session) -> Query: System.privacy_policy.label("system_privacy_policy"), System.vendor_id, PrivacyDeclaration.data_use, - PrivacyDeclaration.legal_basis_for_processing, + legal_basis_override_subquery.c.overridden_legal_basis_for_processing.label( # pylint: disable=no-member + "legal_basis_for_processing" + ), PrivacyDeclaration.features, PrivacyDeclaration.retention_period, + PrivacyDeclaration.purpose, + PrivacyDeclaration.legal_basis_for_processing.label( + "original_legal_basis_for_processing" + ), ) .outerjoin(PrivacyDeclaration, System.id == PrivacyDeclaration.system_id) + .outerjoin( + legal_basis_override_subquery, + legal_basis_override_subquery.c.id == PrivacyDeclaration.id, + ) .filter( or_( + and_(GVL_DATA_USE_FILTER, consent_legal_basis_filter), and_( GVL_DATA_USE_FILTER, - PrivacyDeclaration.legal_basis_for_processing - == LegalBasisForProcessingEnum.CONSENT, - ), - and_( - GVL_DATA_USE_FILTER, - PrivacyDeclaration.legal_basis_for_processing - == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST, + legitimate_interest_legal_basis_filter, NOT_AC_SYSTEM_FILTER, ), AC_SYSTEM_NO_PRIVACY_DECL_FILTER, @@ -201,7 +268,12 @@ def get_matching_privacy_declarations(db: Session) -> Query: matching_privacy_declarations = matching_privacy_declarations.filter( NOT_AC_SYSTEM_FILTER ) - return matching_privacy_declarations + + return ( + matching_privacy_declarations, + consent_legal_basis_filter, + legitimate_interest_legal_basis_filter, + ) def systems_that_match_tcf_data_uses( @@ -238,7 +310,11 @@ def get_relevant_systems_for_tcf_attribute( # pylint: disable=too-many-return-s # For TCF attributes, we need to first filter to systems/privacy declarations that have a relevant GVL data use # as well as a legal basis of processing of consent or legitimate interests - starting_privacy_declarations: Query = get_matching_privacy_declarations(db) + ( + starting_privacy_declarations, + CONSENT_LEGAL_BASIS_FILTER, + LEGITIMATE_INTEREST_LEGAL_BASIS_FILTER, + ) = get_tcf_base_query_and_filters(db) purpose_data_uses: List[str] = [] if tcf_field in [ diff --git a/src/fides/config/consent_settings.py b/src/fides/config/consent_settings.py index 877139659d..a16971b20e 100644 --- a/src/fides/config/consent_settings.py +++ b/src/fides/config/consent_settings.py @@ -15,6 +15,11 @@ class ConsentSettings(FidesSettings): default=False, description="Toggle whether Google AC Mode is enabled." ) + override_vendor_purposes: bool = Field( + default=False, + description="Whether or not vendor purposes can be globally overridden.", + ) + class Config: env_prefix = "FIDES__CONSENT__" @@ -23,8 +28,14 @@ def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: """AC mode only works if TCF mode is also enabled""" tcf_mode = values.get("tcf_enabled") ac_mode = values.get("ac_enabled") + override_vendor_purposes = values.get("override_vendor_purposes") if ac_mode and not tcf_mode: raise ValueError("AC cannot be enabled unless TCF mode is also enabled.") + if override_vendor_purposes and not tcf_mode: + raise ValueError( + "Override vendor purposes cannot be true unless TCF mode is also enabled." + ) + return values diff --git a/tests/conftest.py b/tests/conftest.py index 8a9642bc2a..dd27da6aad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -185,6 +185,14 @@ def enable_ac(config): config.consent.ac_enabled = False +@pytest.fixture(scope="function") +def enable_override_vendor_purposes(config): + assert config.test_mode + config.consent.override_vendor_purposes = True + yield config + config.consent.override_vendor_purposes = False + + @pytest.fixture def loguru_caplog(caplog): handler_id = logger.add(caplog.handler, format="{message}") diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index ba2c1357a5..8b4d987e5f 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -5,10 +5,12 @@ from datetime import datetime from json import loads from typing import Dict, List +from uuid import uuid4 import pytest import requests from fideslang import DEFAULT_TAXONOMY, model_list, models, parse +from fideslang.models import PrivacyDeclaration as PrivacyDeclarationSchema from fideslang.models import System as SystemSchema from pytest import MonkeyPatch from starlette.status import ( @@ -25,10 +27,12 @@ from fides.api.api.v1.endpoints import health from fides.api.db.crud import get_resource +from fides.api.db.system import create_system from fides.api.models.connectionconfig import ConnectionConfig from fides.api.models.datasetconfig import DatasetConfig from fides.api.models.sql_models import Dataset, PrivacyDeclaration, System from fides.api.models.system_history import SystemHistory +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride from fides.api.oauth.roles import OWNER, VIEWER from fides.api.schemas.system import PrivacyDeclarationResponse, SystemResponse from fides.api.schemas.taxonomy_extensions import ( @@ -678,6 +682,9 @@ async def test_system_create( assert privacy_decl.dataset_references == ["another_system_reference"] assert privacy_decl.features == ["Link different devices"] assert privacy_decl.legal_basis_for_processing == "Public interest" + assert ( + await privacy_decl.get_purpose_legal_basis_override() == "Public interest" + ) assert ( privacy_decl.impact_assessment_location == "https://www.example.com/impact_assessment_location" @@ -835,6 +842,7 @@ async def test_system_create_custom_metadata_saas_config( assert result.status_code == HTTP_201_CREATED assert result.json()["name"] == "Test System" assert len(result.json()["privacy_declarations"]) == 2 + assert result.json()["meta"] == { "saas_config": { "type": "stripe", @@ -1593,8 +1601,8 @@ def test_system_update_dictionary_fields( for i, decl in enumerate(system.privacy_declarations): for field in PrivacyDeclarationResponse.__fields__: - decl_val = getattr(decl, field) - if isinstance(decl_val, typing.Hashable): + decl_val = getattr(decl, field, None) + if hasattr(decl, field) and isinstance(decl_val, typing.Hashable): assert decl_val == json_results["privacy_declarations"][i][field] def test_system_update_system_cookies( @@ -2592,3 +2600,224 @@ def test_trailing_slash(test_config: FidesConfig, endpoint_name: str) -> None: assert response.status_code == 200 response = requests.get(f"{url}/", headers=CONFIG.user.auth_header) assert response.status_code == 200 + + +class TestPrivacyDeclarationGetPurposeLegalBasisOverride: + async def test_privacy_declaration_enable_override_is_false( + self, async_session_temp + ): + """Enable override is false so overridden legal basis is going to default + to the defined legal basis""" + resource = SystemSchema( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name=f"test_system_1_{uuid4()}", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="Collect data for content performance", + data_use="analytics.reporting.campaign_insights", + legal_basis_for_processing="Consent", + data_categories=["user"], + ) + ], + ) + + system = await create_system( + resource, async_session_temp, CONFIG.security.oauth_root_client_id + ) + pd = system.privacy_declarations[0] + + assert pd.purpose == 9 + assert await pd.get_purpose_legal_basis_override() == "Consent" + + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + async def test_enable_override_is_true_but_no_matching_purpose( + self, async_session_temp, db + ): + """Privacy Declaration has Special Purpose not Purpose, so no overrides applicable""" + resource = SystemSchema( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name=f"test_system_1_{uuid4()}", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="Collect data for content performance", + data_use="essential.fraud_detection", + legal_basis_for_processing="Consent", + data_categories=["user"], + ) + ], + ) + + system = await create_system( + resource, async_session_temp, CONFIG.security.oauth_root_client_id + ) + pd = system.privacy_declarations[0] + + assert pd.purpose is None + assert await pd.get_purpose_legal_basis_override() == "Consent" + + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + async def test_enable_override_is_true_but_purpose_is_excluded( + self, async_session_temp, db + ): + """Purpose is overridden as excluded, so legal basis returns as None, to match + class-wide override""" + resource = SystemSchema( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name=f"test_system_1_{uuid4()}", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="Collect data for content performance", + data_use="personalize.content.profiling", + legal_basis_for_processing="Consent", + data_categories=["user"], + ) + ], + ) + + system = await create_system( + resource, async_session_temp, CONFIG.security.oauth_root_client_id + ) + pd = system.privacy_declarations[0] + + constraint = TCFPurposeOverride.create( + db, + data={ + "purpose": 5, + "is_included": False, + }, + ) + + assert pd.purpose == 5 + assert await pd.get_purpose_legal_basis_override() is None + + constraint.delete(db) + + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + async def test_legal_basis_is_inflexible(self, async_session_temp, db): + """Purpose is overridden but we can't apply because the legal basis is specified as inflexible""" + resource = SystemSchema( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name=f"test_system_1_{uuid4()}", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="Collect data for content performance", + data_use="personalize.content.profiling", + legal_basis_for_processing="Consent", + flexible_legal_basis_for_processing=False, + data_categories=["user"], + ) + ], + ) + + system = await create_system( + resource, async_session_temp, CONFIG.security.oauth_root_client_id + ) + pd = system.privacy_declarations[0] + + constraint = TCFPurposeOverride.create( + db, + data={ + "purpose": 5, + "is_included": True, + "required_legal_basis": "Legitimate interests", + }, + ) + + assert pd.purpose == 5 + assert await pd.get_purpose_legal_basis_override() == "Consent" + + constraint.delete(db) + + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + async def test_publisher_override_defined_but_no_required_legal_basis_specified( + self, db, async_session_temp + ): + """Purpose override *object* is defined, but no legal basis override""" + resource = SystemSchema( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name=f"test_system_1_{uuid4()}", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="Collect data for content performance", + data_use="analytics.reporting.campaign_insights", + legal_basis_for_processing="Consent", + data_categories=["user"], + ) + ], + ) + + system = await create_system( + resource, async_session_temp, CONFIG.security.oauth_root_client_id + ) + pd = system.privacy_declarations[0] + + constraint = TCFPurposeOverride.create( + db, + data={ + "purpose": 9, + "is_included": True, + }, + ) + + assert pd.purpose == 9 + assert await pd.get_purpose_legal_basis_override() == "Consent" + + constraint.delete(db) + + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", + ) + async def test_publisher_override_defined_with_required_legal_basis_specified( + self, async_session_temp, db + ): + """Purpose override specified along with the requirements to apply that override""" + resource = SystemSchema( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name=f"test_system_1_{uuid4()}", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="Collect data for content performance", + data_use="functional.service.improve", + legal_basis_for_processing="Consent", + data_categories=["user"], + ) + ], + ) + + system = await create_system( + resource, async_session_temp, CONFIG.security.oauth_root_client_id + ) + override = TCFPurposeOverride.create( + db, + data={ + "purpose": 10, + "is_included": True, + "required_legal_basis": "Legitimate interests", + }, + ) + pd = system.privacy_declarations[0] + + assert pd.purpose == 10 + assert await pd.get_purpose_legal_basis_override() == "Legitimate interests" + + override.delete(db) diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index 04d40f6085..32902f937e 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -345,6 +345,51 @@ def test_scan_system_okta_fail( ) +class TestPrivacyDeclarationPurpose: + def test_privacy_declaration_purpose(self): + pd = PrivacyDeclaration( + name="declaration-name", + data_categories=[], + data_use="analytics.reporting.campaign_insights", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ingress=None, + egress=None, + ) + + assert pd.purpose == 9 + + def test_privacy_declaration_special_purpose(self): + """Special purposes are not returned under the purpose hybrid property""" + pd = PrivacyDeclaration( + name="declaration-name", + data_categories=[], + data_use="essential.service.security", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ingress=None, + egress=None, + ) + + assert pd.purpose is None + + def test_privacy_declaration_non_tcf_data_use(self): + pd = PrivacyDeclaration( + name="declaration-name", + data_categories=[], + data_use="essential", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ingress=None, + egress=None, + ) + + assert pd.purpose is None + + class TestUpsertCookies: @pytest.fixture() async def test_cookie_system( diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 252e4d51b3..9d193ca83f 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -68,6 +68,7 @@ _create_local_default_storage, default_storage_config_name, ) +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride from fides.api.oauth.roles import APPROVER, VIEWER from fides.api.schemas.messaging.messaging import ( MessagingServiceDetails, @@ -3133,3 +3134,17 @@ def skimbit_system(db): }, ) return system + + +@pytest.fixture(scope="function") +def purpose_three_consent_publisher_override(db): + override = TCFPurposeOverride.create( + db, + data={ + "purpose": 3, + "is_included": True, + "required_legal_basis": "Consent", + }, + ) + yield override + override.delete(db) diff --git a/tests/ops/api/v1/endpoints/test_config_endpoints.py b/tests/ops/api/v1/endpoints/test_config_endpoints.py index a24c094d02..da54a35ed7 100644 --- a/tests/ops/api/v1/endpoints/test_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_config_endpoints.py @@ -970,7 +970,9 @@ def test_get_config( } for key in config.keys(): - assert (key in allowed_top_level_config_keys), "Unexpected config API change, please review with Ethyca security team" + assert ( + key in allowed_top_level_config_keys + ), "Unexpected config API change, please review with Ethyca security team" assert "security" in config assert "user" in config diff --git a/tests/ops/models/test_privacy_preference.py b/tests/ops/models/test_privacy_preference.py index 2a51ec7b04..2751e244f5 100644 --- a/tests/ops/models/test_privacy_preference.py +++ b/tests/ops/models/test_privacy_preference.py @@ -1653,6 +1653,48 @@ def test_determine_relevant_systems_for_ac_system_purpose_legitimate_interests( == [] ) + @pytest.mark.usefixtures( + "enable_override_vendor_purposes", "purpose_three_consent_publisher_override" + ) + def test_determine_relevant_systems_for_with_purpose_override( + self, + db, + system_with_no_uses, + ): + """Relevant system calculation takes into account legal basis overrides""" + # Add data use to system that corresponds to purpose 3. Also has LI legal basis, but override sets it + # to Consent + pd_1 = PrivacyDeclaration.create( + db=db, + data={ + "name": "Collect data for content performance", + "system_id": system_with_no_uses.id, + "data_categories": ["user.device.cookie_id"], + "data_use": "marketing.advertising.profiling", + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "data_subjects": ["customer"], + "dataset_references": None, + "legal_basis_for_processing": "Legitimate interests", + "egress": None, + "ingress": None, + }, + ) + + assert PrivacyPreferenceHistory.determine_relevant_systems( + db, tcf_field=TCFComponentType.purpose_consent.value, tcf_value=3 + ) == [system_with_no_uses.fides_key] + + assert ( + PrivacyPreferenceHistory.determine_relevant_systems( + db, + tcf_field=TCFComponentType.purpose_legitimate_interests.value, + tcf_value=3, + ) + == [] + ) + + pd_1.delete(db) + class TestCurrentPrivacyPreference: def test_get_preference_by_notice_and_fides_user_device( diff --git a/tests/ops/util/test_consent_util.py b/tests/ops/util/test_consent_util.py index 33d594506f..6c4cbd7184 100644 --- a/tests/ops/util/test_consent_util.py +++ b/tests/ops/util/test_consent_util.py @@ -30,6 +30,7 @@ add_errored_system_status_for_consent_reporting, cache_initial_status_and_identities_for_consent_reporting, create_default_experience_config, + create_default_tcf_purpose_overrides_on_startup, create_privacy_notices_util, create_tcf_experiences_on_startup, get_fides_user_device_id_provided_identity, @@ -1289,3 +1290,15 @@ def test_create_tcf_experiences_on_startup(self, db): experience_config = be_exp.experience_config assert experience_config.is_default assert experience_config.component == ComponentType.tcf_overlay + + +class TestLoadTCFPurposeOverrides: + def test_load_tcf_purpose_overrides_on_startup(self, db): + """Sanity check on creating TCF purpose overrides""" + default_override_objects_added = ( + create_default_tcf_purpose_overrides_on_startup(db) + ) + assert len(default_override_objects_added) == 11 + for override in default_override_objects_added: + assert override.is_included is True + assert override.required_legal_basis is None diff --git a/tests/ops/util/test_tcf_privacy_declarations.py b/tests/ops/util/test_tcf_privacy_declarations.py new file mode 100644 index 0000000000..bce95dd7e7 --- /dev/null +++ b/tests/ops/util/test_tcf_privacy_declarations.py @@ -0,0 +1,144 @@ +import pytest + +from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride +from fides.api.util.tcf.tcf_experience_contents import get_tcf_base_query_and_filters + + +class TestBaseTCFQuery: + """Tests matching privacy declarations returned that are the basis of the TCF Experience and the "relevant_systems" + that are saved for consent reporting + """ + + @pytest.mark.usefixtures( + "emerse_system", + ) + def test_get_matching_privacy_declarations_enable_purpose_override_is_false( + self, db + ): + """No legal bases are overridden because this features is off""" + declarations, _, _ = get_tcf_base_query_and_filters(db) + + assert declarations.count() == 13 + + mapping = { + declaration.data_use: declaration.purpose for declaration in declarations + } + + # marketing.advertising.serving, essential.service.security, essential.fraud_detection map to special purposes, not purposes + assert mapping == { + "marketing.advertising.serving": None, + "essential.service.security": None, + "essential.fraud_detection": None, + "analytics.reporting.campaign_insights": 9, + "analytics.reporting.content_performance": 8, + "analytics.reporting.ad_performance": 7, + "marketing.advertising.frequency_capping": 2, + "marketing.advertising.first_party.contextual": 2, + "marketing.advertising.negative_targeting": 2, + "marketing.advertising.first_party.targeted": 4, + "marketing.advertising.third_party.targeted": 4, + "marketing.advertising.profiling": 3, + "functional.storage": 1, + } + + @pytest.mark.usefixtures("enable_override_vendor_purposes") + def test_privacy_declaration_purpose_overrides(self, db, emerse_system): + """Comprehensive test of the various scenarios for privacy declaration purpose overrides when building the + base TCF query + + Some of the specifics aren't configurations that would be allowed for emerse, but ignore that here. + """ + + # As some test setup, override purpose 7 declaration to have an inflexible legal basis + purpose_7_decl = next( + decl + for decl in emerse_system.privacy_declarations + if decl.data_use == "analytics.reporting.ad_performance" + ) + purpose_7_decl.flexible_legal_basis_for_processing = False + purpose_7_decl.save(db) + + # For more test setup, create some purpose overrides + + # Purpose override that matches the legal basis already on the system's declaration + TCFPurposeOverride.create( + db, + data={"purpose": 1, "is_included": True, "required_legal_basis": "Consent"}, + ) + + # Purpose override that marks Purpose 2 as excluded + TCFPurposeOverride.create( + db, + data={ + "purpose": 2, + "is_included": False, + }, + ) + + # Purpose override object defined, but no legal basis override specified + TCFPurposeOverride.create( + db, + data={ + "purpose": 3, + "is_included": True, + "required_legal_basis": None, + }, + ) + + # Purpose override defined with different legal basis than the one on the System's declaration + TCFPurposeOverride.create( + db, + data={ + "purpose": 4, + "is_included": True, + "required_legal_basis": "Legitimate interests", + }, + ) + + # Purpose override defined with differing legal basis, however, purpose 7 above was marked as inflexible + TCFPurposeOverride.create( + db, + data={ + "purpose": 7, + "is_included": True, + "required_legal_basis": "Consent", + }, + ) + + declarations, _, _ = get_tcf_base_query_and_filters(db) + + legal_basis_overrides = { + declaration.purpose: declaration.legal_basis_for_processing + for declaration in declarations + if declaration.purpose + } + + # Purpose 1 had no change, because the override matched + # Purpose 2 has been removed altogether + # Purpose 4 Legal Basis has been overridden to Legitimate Interests legal basis + # Purpose 7's override wasn't applied because that declaration was marked as inflexible. + assert legal_basis_overrides == { + 9: "Legitimate interests", + 8: "Legitimate interests", + 7: "Legitimate interests", + 4: "Legitimate interests", + 3: "Consent", + 1: "Consent", + } + + original_legal_basis = { + declaration.purpose: declaration.original_legal_basis_for_processing + for declaration in declarations + if declaration.purpose + } + assert original_legal_basis == { + 9: "Legitimate interests", + 8: "Legitimate interests", + 7: "Legitimate interests", + 4: "Consent", + 3: "Consent", + 1: "Consent", + } + + # The three declarations on Emerse with data uses mapping to Purpose 2 have been excluded + assert declarations.count() == 10