From 09c29e8afb91a9b52f7b2664433d3342eb15ba50 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 20 Oct 2023 16:41:27 -0500 Subject: [PATCH 1/3] Update the contents of the version hash that is used to determine when consent should be resurfaced. Previously we hashed purpose x legal basis and vendor x legal basis to match what could be saved in the string. This largely updates to save three dimensions together vendor x legal basis x purpose New sections are: - Policy version (no change) - Vendor purpose consents - Vendor purpose legitimate interests - gvl vendors disclosed --- src/fides/api/util/tcf/experience_meta.py | 107 ++++++++-- tests/fixtures/application_fixtures.py | 2 +- .../test_privacy_experience_endpoints.py | 2 +- tests/ops/util/test_tc_string.py | 195 ++++++++++++++---- 4 files changed, 246 insertions(+), 60 deletions(-) diff --git a/src/fides/api/util/tcf/experience_meta.py b/src/fides/api/util/tcf/experience_meta.py index 18c5c344b1..b5e54279c3 100644 --- a/src/fides/api/util/tcf/experience_meta.py +++ b/src/fides/api/util/tcf/experience_meta.py @@ -1,32 +1,49 @@ import hashlib import json -from typing import Dict, List +from typing import Dict, List, Union -from pydantic import Extra, root_validator +from pydantic import Extra, Field, root_validator from fides.api.models.privacy_notice import UserConsentPreference from fides.api.schemas.base_class import FidesSchema from fides.api.schemas.privacy_experience import ExperienceMeta -from fides.api.schemas.tcf import TCMobileData +from fides.api.schemas.tcf import ( + TCFVendorConsentRecord, + TCFVendorLegitimateInterestsRecord, + TCMobileData, +) from fides.api.util.tcf.tc_mobile_data import build_tc_data_for_mobile from fides.api.util.tcf.tc_model import TCModel, convert_tcf_contents_to_tc_model from fides.api.util.tcf.tcf_experience_contents import TCFExperienceContents class TCFVersionHash(FidesSchema): - """Minimal subset of the TCF experience details that capture when consent should be resurfaced""" + """Minimal subset of the TCF experience details that capture when consent should be resurfaced - policy_version: int - purpose_consents: List[int] - purpose_legitimate_interests: List[int] - special_feature_optins: List[int] - vendor_consents: List[int] - vendor_legitimate_interests: List[int] + These values are later hashed, and you can treat a change in a hash as a trigger for re-establishing + transparency and consent + """ + + policy_version: int = Field( + description="TCF Policy Version. A new policy version invalidates existing strings and requires CMPs to " + "re-establish transparency and consent from users." + ) + vendor_purpose_consents: List[str] = Field( + description="Stores vendor * purpose * consent legal basis in the format " + "-" + ) + vendor_purpose_legitimate_interests: List[str] = Field( + description="Stores vendor * purpose * legitimate interest legal basis in the format" + " -" + ) + gvl_vendors_disclosed: List[int] = Field( + description="List of GVL vendors disclosed from the last vendor list" + ) @root_validator() @classmethod def sort_lists(cls, values: Dict) -> Dict: - """Verify lists are sorted ascending for repeatability""" + """Verify lists are sorted for repeatability""" for field, val in values.items(): if isinstance(val, list): values[field] = sorted(val) @@ -46,18 +63,74 @@ def _build_tcf_version_hash_model( get the maximum possible number of attributes added to our hash for the current system configuration. """ + + def build_vendor_by_purpose_strings( + vendor_list: Union[ + List[TCFVendorConsentRecord], List[TCFVendorLegitimateInterestsRecord] + ] + ) -> List[str]: + """Hash helper for building vendor x purposes. Converts a list of vendor records + into a list of strings each consisting of a vendor id, a separator (-), and comma + separated purposes (if applicable) with the given legitimate interest. + + Example: + [-] + ["gacp.100", "gvl.8-1,2,3", "gvl.56-3,4,5] + """ + vendor_by_purpose_lists: List[str] = [] + for vendor_record in vendor_list: + vendor_id: str = vendor_record.id + purposes_key: str = ( + "purpose_consents" + if isinstance(vendor_record, TCFVendorConsentRecord) + else "purpose_legitimate_interests" + ) + purpose_ids = sorted( + purpose_record.id + for purpose_record in getattr(vendor_record, purposes_key) + ) + concatenated_purposes_string: str = ",".join( + str(purpose_id) for purpose_id in purpose_ids + ) + vendor_by_purpose_lists.append( + vendor_id + "-" + concatenated_purposes_string + if concatenated_purposes_string + else vendor_id + ) + return vendor_by_purpose_lists + + vendor_consents: List[str] = build_vendor_by_purpose_strings( + tcf_contents.tcf_vendor_consents + tcf_contents.tcf_system_consents + ) + vendor_li: List[str] = build_vendor_by_purpose_strings( + tcf_contents.tcf_vendor_legitimate_interests + + tcf_contents.tcf_system_legitimate_interests + ) + + # Building the TC model to pull off the GVL Policy version and get vendors_disclosed + # which is a component of the TC string. model: TCModel = convert_tcf_contents_to_tc_model( tcf_contents, UserConsentPreference.opt_in ) - return TCFVersionHash(**model.dict()) + return TCFVersionHash( + policy_version=model.policy_version, + vendor_purpose_consents=vendor_consents, + vendor_purpose_legitimate_interests=vendor_li, + gvl_vendors_disclosed=model.vendors_disclosed, + ) -def build_tcf_version_hash(tcf_contents: TCFExperienceContents) -> str: - """Returns a 12-character version hash for TCF that should only change - if there are updates to vendors, purposes, and special features sections or legal basis. - This hash can be used to determine if the TCF Experience needs to be resurfaced to the customer, - because the experience has changed in a meaningful way. +def build_tcf_version_hash(tcf_contents: TCFExperienceContents) -> str: + """Returns a 12-character version hash of the TCF Experience that changes when consent should be recollected. + + Changes when: + - New GVL, AC Vendor, or "other" vendor is added to TCF Experience + - Existing vendor adds a new TCF purpose + - Existing vendor changes the legal basis for a previously-disclosed purpose + - TCF Policy version changes + - Vendor moves from AC to GVL + - Vendor moves from GVL 2 to GVL 3 """ tcf_version_hash_model: TCFVersionHash = _build_tcf_version_hash_model(tcf_contents) json_str: str = json.dumps(tcf_version_hash_model.dict(), sort_keys=True) diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 235ca66f02..9ed2ec5d8f 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -2869,7 +2869,7 @@ def ac_system_without_privacy_declaration(db: Session) -> System: data={ "fides_key": f"ac_system{uuid.uuid4()}", "vendor_id": "gacp.100", - "name": f"Test AC System", + "name": f"Test AC System 2", "organization_fides_key": "default_organization", "system_type": "Service", }, diff --git a/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py index ff299b19d3..0c24d59cba 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py @@ -823,7 +823,7 @@ def test_tcf_enabled_with_overlapping_vendors( assert resp.json()["items"][0]["tcf_system_legitimate_interests"] == [] assert resp.json()["items"][0]["gvl"]["gvlSpecificationVersion"] == 3 meta = resp.json()["items"][0]["meta"] - assert meta["version_hash"] == "f2db7626ca0b" + assert meta["version_hash"] == "dbde7265d5dd" assert meta["accept_all_fides_string"] assert meta["accept_all_fides_mobile_data"] assert meta["reject_all_fides_string"] diff --git a/tests/ops/util/test_tc_string.py b/tests/ops/util/test_tc_string.py index 9f50e551c9..699b503c25 100644 --- a/tests/ops/util/test_tc_string.py +++ b/tests/ops/util/test_tc_string.py @@ -38,31 +38,78 @@ def test_build_tcf_version_hash_model(self, db): version_hash_model = _build_tcf_version_hash_model(tcf_contents=tcf_contents) assert version_hash_model == TCFVersionHash( policy_version=4, - purpose_consents=[8], - purpose_legitimate_interests=[], - special_feature_optins=[], - vendor_consents=[42], - vendor_legitimate_interests=[], + vendor_purpose_consents=["gvl.42-8"], + vendor_purpose_legitimate_interests=[], + gvl_vendors_disclosed=[42], ) version_hash = build_tcf_version_hash(tcf_contents) - assert version_hash == "f2db7626ca0b" + assert version_hash == "dbde7265d5dd" - def test_version_hash_model_sorts_ascending(self): + def test_version_hash_model_sorting(self): + """We're just doing string sorting here, we don't need to do natural language + sorting, as long as it's repeatable""" version_hash_model = TCFVersionHash( policy_version=4, - purpose_consents=[5, 4, 3, 1], - purpose_legitimate_interests=[7, 8], - special_feature_optins=[2, 1], - vendor_consents=[8, 2, 1], - vendor_legitimate_interests=[141, 14, 1], + vendor_purpose_consents=[ + "gvl.8-1,4,6,7", + "gvl.39-1,4,6,7", + "gvl.5-1,2,3,4", + "gvl.1", + "gacp.4" "gacp.10,1", + "gacp.1", + "ctl_3c809e3f-96b3-4aec-a0c6-f3904104559b-9", + ], + vendor_purpose_legitimate_interests=["gvl.39-1,4,6,7", "gvl.100-1,2"], + gvl_vendors_disclosed=[100, 39, 5, 8, 1], ) assert version_hash_model.policy_version == 4 - assert version_hash_model.purpose_consents == [1, 3, 4, 5] - assert version_hash_model.purpose_legitimate_interests == [7, 8] - assert version_hash_model.special_feature_optins == [1, 2] - assert version_hash_model.vendor_legitimate_interests == [1, 14, 141] + assert version_hash_model.vendor_purpose_consents == [ + "ctl_3c809e3f-96b3-4aec-a0c6-f3904104559b-9", + "gacp.1", + "gacp.4gacp.10,1", + "gvl.1", + "gvl.39-1,4,6,7", + "gvl.5-1,2,3,4", + "gvl.8-1,4,6,7", + ] + assert version_hash_model.vendor_purpose_legitimate_interests == [ + "gvl.100-1,2", + "gvl.39-1,4,6,7", + ] + assert version_hash_model.gvl_vendors_disclosed == [1, 5, 8, 39, 100] + + @pytest.mark.usefixtures( + "skimbit_system", + "emerse_system", + "captify_technologies_system", + "ac_system_without_privacy_declaration", + "ac_system_with_privacy_declaration", + ) + def test_vendor_hash_model_contents(self, db, system): + """Test building hash model with a mixture of GVL, AC, and regular Systems without a vendor""" + decl = system.privacy_declarations[0] + decl.legal_basis_for_processing = "Legitimate interests" + decl.data_use = "marketing.advertising.first_party.targeted" + decl.save(db) + + tcf_contents = get_tcf_contents(db) + version_hash_model = _build_tcf_version_hash_model(tcf_contents=tcf_contents) + + assert version_hash_model.policy_version == 4 + assert version_hash_model.vendor_purpose_consents == [ + "gacp.100", + "gacp.8-1", + "gvl.2-1,2,3,4,7,9,10", + "gvl.8-1,3,4", + ] + assert version_hash_model.vendor_purpose_legitimate_interests == [ + system.id + "-4", + "gvl.46-7,8,10", + "gvl.8-2,7,8,9", + ] + assert version_hash_model.gvl_vendors_disclosed == [2, 8, 46] @pytest.mark.usefixtures("captify_technologies_system") def test_build_tcf_version_hash_removing_declaration( @@ -72,15 +119,13 @@ def test_build_tcf_version_hash_removing_declaration( version_hash_model = _build_tcf_version_hash_model(tcf_contents=tcf_contents) assert version_hash_model == TCFVersionHash( policy_version=4, - purpose_consents=[1, 2, 3, 4, 7, 9, 10], - purpose_legitimate_interests=[], - special_feature_optins=[2], - vendor_consents=[2], - vendor_legitimate_interests=[], + vendor_purpose_consents=["gvl.2-1,2,3,4,7,9,10"], + vendor_purpose_legitimate_interests=[], + gvl_vendors_disclosed=[2], ) version_hash = build_tcf_version_hash(tcf_contents) - assert version_hash == "eaab1c195073" + assert version_hash == "0429105be515" # Remove the privacy declaration corresponding to purpose 1 for decl in captify_technologies_system.privacy_declarations: @@ -92,30 +137,61 @@ def test_build_tcf_version_hash_removing_declaration( version_hash_model = _build_tcf_version_hash_model(tcf_contents=tcf_contents) assert version_hash_model == TCFVersionHash( policy_version=4, - purpose_consents=[2, 3, 4, 7, 9, 10], - purpose_legitimate_interests=[], - special_feature_optins=[2], - vendor_consents=[2], - vendor_legitimate_interests=[], + vendor_purpose_consents=["gvl.2-2,3,4,7,9,10"], + vendor_purpose_legitimate_interests=[], + gvl_vendors_disclosed=[2], + ) + + version_hash = build_tcf_version_hash(tcf_contents) + assert version_hash == "f2e19b4b3eae" + + def test_build_tcf_version_hash_adding_vendor(self, db, system): + system.vendor_id = "gvl.88" + system.save(db) + + tcf_contents = get_tcf_contents(db) + version_hash_model = _build_tcf_version_hash_model(tcf_contents=tcf_contents) + assert version_hash_model == TCFVersionHash( + policy_version=4, + vendor_purpose_consents=[], + vendor_purpose_legitimate_interests=[], + gvl_vendors_disclosed=[], ) version_hash = build_tcf_version_hash(tcf_contents) - assert version_hash == "77ed45ac8d43" + assert version_hash == "6b2062179826" + + # Add declaration to system with TCF purpose, so now system shows up + decl = system.privacy_declarations[0] + decl.legal_basis_for_processing = "Legitimate interests" + decl.data_use = "marketing.advertising.first_party.targeted" + decl.save(db) - def test_build_tcf_version_hash_adding_data_use(self, db, emerse_system): + # Recalculate version hash model and version tcf_contents = get_tcf_contents(db) version_hash_model = _build_tcf_version_hash_model(tcf_contents=tcf_contents) assert version_hash_model == TCFVersionHash( policy_version=4, - purpose_consents=[1, 3, 4], - purpose_legitimate_interests=[2, 7, 8, 9], - special_feature_optins=[], - vendor_consents=[8], - vendor_legitimate_interests=[8], + vendor_purpose_consents=[], + vendor_purpose_legitimate_interests=[f"gvl.88-4"], + gvl_vendors_disclosed=[], ) version_hash = build_tcf_version_hash(tcf_contents) - assert version_hash == "a2e85860c68b" + assert version_hash == "c5a4e3b672e3" + + def test_build_tcf_version_hash_adding_purpose(self, db, emerse_system): + tcf_contents = get_tcf_contents(db) + version_hash_model = _build_tcf_version_hash_model(tcf_contents=tcf_contents) + assert version_hash_model == TCFVersionHash( + policy_version=4, + vendor_purpose_consents=["gvl.8-1,3,4"], + vendor_purpose_legitimate_interests=["gvl.8-2,7,8,9"], + gvl_vendors_disclosed=[8], + ) + + version_hash = build_tcf_version_hash(tcf_contents) + assert version_hash == "3a655ef13ee6" # Adding privacy declaration for purpose 10 PrivacyDeclaration.create( @@ -136,15 +212,52 @@ def test_build_tcf_version_hash_adding_data_use(self, db, emerse_system): version_hash_model = _build_tcf_version_hash_model(tcf_contents=tcf_contents) assert version_hash_model == TCFVersionHash( policy_version=4, - purpose_consents=[1, 3, 4, 10], - purpose_legitimate_interests=[2, 7, 8, 9], - special_feature_optins=[], - vendor_consents=[8], - vendor_legitimate_interests=[8], + vendor_purpose_consents=["gvl.8-1,3,4,10"], + vendor_purpose_legitimate_interests=["gvl.8-2,7,8,9"], + gvl_vendors_disclosed=[8], + ) + + version_hash = build_tcf_version_hash(tcf_contents) + assert version_hash == "c290c3d76a26" + + def test_build_tcf_version_hash_updating_legal_basis(self, db, system): + system.vendor_id = "gvl.88" + system.save(db) + + # Add declaration to system with TCF purpose, so now system shows up + decl = system.privacy_declarations[0] + decl.legal_basis_for_processing = "Legitimate interests" + decl.data_use = "marketing.advertising.first_party.targeted" + decl.save(db) + + tcf_contents = get_tcf_contents(db) + version_hash_model = _build_tcf_version_hash_model(tcf_contents=tcf_contents) + assert version_hash_model == TCFVersionHash( + policy_version=4, + vendor_purpose_consents=[], + vendor_purpose_legitimate_interests=[f"gvl.88-4"], + gvl_vendors_disclosed=[], + ) + + version_hash = build_tcf_version_hash(tcf_contents) + assert version_hash == "c5a4e3b672e3" + + # Update legal basis + decl.legal_basis_for_processing = "Consent" + decl.save(db) + + # Recalculate version hash model and version + tcf_contents = get_tcf_contents(db) + version_hash_model = _build_tcf_version_hash_model(tcf_contents=tcf_contents) + assert version_hash_model == TCFVersionHash( + policy_version=4, + vendor_purpose_consents=[f"gvl.88-4"], + vendor_purpose_legitimate_interests=[], + gvl_vendors_disclosed=[], ) version_hash = build_tcf_version_hash(tcf_contents) - assert version_hash == "73c0762c9442" + assert version_hash == "a279467aec23" class TestBuildTCModel: From d39ed31f4e9866c7c6469faaa249e7d854715df7 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 20 Oct 2023 16:45:30 -0500 Subject: [PATCH 2/3] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 901a0f0b51..e8f8f8b654 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The types of changes are: ### Added - Added a `FidesPreferenceToggled` event to Fides.js to track when user preferences change without being saved [#4253](https://github.com/ethyca/fides/pull/4253) - Add AC Systems to the TCF Overlay under Vendor Consents section [#4266](https://github.com/ethyca/fides/pull/4266/) +- Update Version Hash Contents [#4313](https://github.com/ethyca/fides/pull/4313) ### Changed - Derive cookie storage info, privacy policy and legitimate interest disclosure URLs, and data retention data from the data map instead of directly from gvl.json [#4286](https://github.com/ethyca/fides/pull/4286) From a0a6a6c7a337b72dd730089be41af9705851d0f6 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 24 Oct 2023 12:03:18 -0500 Subject: [PATCH 3/3] Update docstrings from CR and update test that enables AC now that flag is needed for AC vendors to show up in vendor hash. --- src/fides/api/util/tcf/experience_meta.py | 15 ++++++++------- tests/ops/util/test_tc_string.py | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/fides/api/util/tcf/experience_meta.py b/src/fides/api/util/tcf/experience_meta.py index bb9941a5a9..c11e1c07d6 100644 --- a/src/fides/api/util/tcf/experience_meta.py +++ b/src/fides/api/util/tcf/experience_meta.py @@ -30,21 +30,21 @@ class TCFVersionHash(FidesSchema): "re-establish transparency and consent from users." ) vendor_purpose_consents: List[str] = Field( - description="Stores vendor * purpose * consent legal basis in the format " + description="Stores vendor * purpose * consent legal basis in the format: " "-" ) vendor_purpose_legitimate_interests: List[str] = Field( - description="Stores vendor * purpose * legitimate interest legal basis in the format" - " -" + description="Stores vendor * purpose * legitimate interest legal basis in the format: " + "-" ) gvl_vendors_disclosed: List[int] = Field( - description="List of GVL vendors disclosed from the last vendor list" + description="List of GVL vendors disclosed from the current vendor list" ) @root_validator() @classmethod def sort_lists(cls, values: Dict) -> Dict: - """Verify lists are sorted for repeatability""" + """Verify lists are sorted deterministically for repeatability""" for field, val in values.items(): if isinstance(val, list): values[field] = sorted(val) @@ -72,7 +72,7 @@ def build_vendor_by_purpose_strings( ) -> List[str]: """Hash helper for building vendor x purposes. Converts a list of vendor records into a list of strings each consisting of a vendor id, a separator (-), and comma - separated purposes (if applicable) with the given legitimate interest. + separated purposes (if applicable) with the given legal basis. Example: [-] @@ -118,7 +118,8 @@ def build_vendor_by_purpose_strings( policy_version=model.policy_version, vendor_purpose_consents=vendor_consents, vendor_purpose_legitimate_interests=vendor_li, - gvl_vendors_disclosed=model.vendors_disclosed, + gvl_vendors_disclosed=model.vendors_disclosed, # `vendors_disclosed` field only has GVL vendors present + # in "current" GVL vendor list ) diff --git a/tests/ops/util/test_tc_string.py b/tests/ops/util/test_tc_string.py index 419946cbb7..6cc2e349c1 100644 --- a/tests/ops/util/test_tc_string.py +++ b/tests/ops/util/test_tc_string.py @@ -87,6 +87,7 @@ def test_version_hash_model_sorting(self): "captify_technologies_system", "ac_system_without_privacy_declaration", "ac_system_with_privacy_declaration", + "enable_ac", ) def test_vendor_hash_model_contents(self, db, system): """Test building hash model with a mixture of GVL, AC, and regular Systems without a vendor"""