Update Version Hash Contents (#4313)

- Update the contents of the version hash that is used to determine when consent should be resurfaced for TCF. 
    - This largely updates to save three dimensions together vendor x legal basis x purpose and also includes AC vendors and other applicable systems in the hash, as well as a determination of whether vendors moved from GVL 2 to GVL 3

CHANGELOG.md

@@ -33,6 +33,7 @@ The types of changes are:
 ### Changed
 - Derive cookie storage info, privacy policy and legitimate interest disclosure URLs, and data retention data from the data map instead of directly from gvl.json [#4286](https://github.com/ethyca/fides/pull/4286)
 - Updated TCF Version for backend consent reporting [#4305](https://github.com/ethyca/fides/pull/4305)
+- Update Version Hash Contents [#4313](https://github.com/ethyca/fides/pull/4313)
 
 ## [2.22.1](https://github.com/ethyca/fides/compare/2.22.0...2.22.1)
 

src/fides/api/util/tcf/experience_meta.py

@@ -1,33 +1,50 @@
 import hashlib
 import json
-from typing import Dict, List
+from typing import Dict, List, Union
 
-from pydantic import Extra, root_validator
+from pydantic import Extra, Field, root_validator
 
 from fides.api.models.privacy_notice import UserConsentPreference
 from fides.api.schemas.base_class import FidesSchema
 from fides.api.schemas.privacy_experience import ExperienceMeta
-from fides.api.schemas.tcf import TCMobileData
+from fides.api.schemas.tcf import (
+    TCFVendorConsentRecord,
+    TCFVendorLegitimateInterestsRecord,
+    TCMobileData,
+)
 from fides.api.util.tcf.fides_string import build_fides_string
 from fides.api.util.tcf.tc_mobile_data import build_tc_data_for_mobile
 from fides.api.util.tcf.tc_model import TCModel, convert_tcf_contents_to_tc_model
 from fides.api.util.tcf.tcf_experience_contents import TCFExperienceContents
 
 
 class TCFVersionHash(FidesSchema):
-    """Minimal subset of the TCF experience details that capture when consent should be resurfaced"""
+    """Minimal subset of the TCF experience details that capture when consent should be resurfaced
 
-    policy_version: int
-    purpose_consents: List[int]
-    purpose_legitimate_interests: List[int]
-    special_feature_optins: List[int]
-    vendor_consents: List[int]
-    vendor_legitimate_interests: List[int]
+    These values are later hashed, and you can treat a change in a hash as a trigger for re-establishing
+    transparency and consent
+    """
+
+    policy_version: int = Field(
+        description="TCF Policy Version. A new policy version invalidates existing strings and requires CMPs to "
+        "re-establish transparency and consent from users."
+    )
+    vendor_purpose_consents: List[str] = Field(
+        description="Stores vendor * purpose * consent legal basis in the format: "
+        "<universal_vendor_id>-<comma separated purposes>"
+    )
+    vendor_purpose_legitimate_interests: List[str] = Field(
+        description="Stores vendor * purpose * legitimate interest legal basis in the format: "
+        "<universal_vendor_id>-<comma separated purposes>"
+    )
+    gvl_vendors_disclosed: List[int] = Field(
+        description="List of GVL vendors disclosed from the current vendor list"
+    )
 
     @root_validator()
     @classmethod
     def sort_lists(cls, values: Dict) -> Dict:
-        """Verify lists are sorted ascending for repeatability"""
+        """Verify lists are sorted deterministically for repeatability"""
         for field, val in values.items():
             if isinstance(val, list):
                 values[field] = sorted(val)
@@ -47,18 +64,75 @@ def _build_tcf_version_hash_model(
     get the maximum possible number of attributes added to our hash for the current
     system configuration.
     """
+
+    def build_vendor_by_purpose_strings(
+        vendor_list: Union[
+            List[TCFVendorConsentRecord], List[TCFVendorLegitimateInterestsRecord]
+        ]
+    ) -> List[str]:
+        """Hash helper for building vendor x purposes.  Converts a list of vendor records
+        into a list of strings each consisting of a vendor id, a separator (-), and comma
+        separated purposes (if applicable) with the given legal basis.
+
+        Example:
+        [<universal_vendor_id>-<comma separated purposes>]
+        ["gacp.100", "gvl.8-1,2,3", "gvl.56-3,4,5]
+        """
+        vendor_by_purpose_lists: List[str] = []
+        for vendor_record in vendor_list:
+            vendor_id: str = vendor_record.id
+            purposes_key: str = (
+                "purpose_consents"
+                if isinstance(vendor_record, TCFVendorConsentRecord)
+                else "purpose_legitimate_interests"
+            )
+            purpose_ids = sorted(
+                purpose_record.id
+                for purpose_record in getattr(vendor_record, purposes_key)
+            )
+            concatenated_purposes_string: str = ",".join(
+                str(purpose_id) for purpose_id in purpose_ids
+            )
+            vendor_by_purpose_lists.append(
+                vendor_id + "-" + concatenated_purposes_string
+                if concatenated_purposes_string
+                else vendor_id
+            )
+        return vendor_by_purpose_lists
+
+    vendor_consents: List[str] = build_vendor_by_purpose_strings(
+        tcf_contents.tcf_vendor_consents + tcf_contents.tcf_system_consents
+    )
+    vendor_li: List[str] = build_vendor_by_purpose_strings(
+        tcf_contents.tcf_vendor_legitimate_interests
+        + tcf_contents.tcf_system_legitimate_interests
+    )
+
+    # Building the TC model to pull off the GVL Policy version and get vendors_disclosed
+    # which is a component of the TC string.
     model: TCModel = convert_tcf_contents_to_tc_model(
         tcf_contents, UserConsentPreference.opt_in
     )
-    return TCFVersionHash(**model.dict())
 
+    return TCFVersionHash(
+        policy_version=model.policy_version,
+        vendor_purpose_consents=vendor_consents,
+        vendor_purpose_legitimate_interests=vendor_li,
+        gvl_vendors_disclosed=model.vendors_disclosed,  # `vendors_disclosed` field only has GVL vendors present
+        # in "current" GVL vendor list
+    )
 
-def build_tcf_version_hash(tcf_contents: TCFExperienceContents) -> str:
-    """Returns a 12-character version hash for TCF that should only change
-    if there are updates to vendors, purposes, and special features sections or legal basis.
 
-    This hash can be used to determine if the TCF Experience needs to be resurfaced to the customer,
-    because the experience has changed in a meaningful way.
+def build_tcf_version_hash(tcf_contents: TCFExperienceContents) -> str:
+    """Returns a 12-character version hash of the TCF Experience that changes when consent should be recollected.
+
+    Changes when:
+      - New GVL, AC Vendor, or "other" vendor is added to TCF Experience
+      - Existing vendor adds a new TCF purpose
+      - Existing vendor changes the legal basis for a previously-disclosed purpose
+      - TCF Policy version changes
+      - Vendor moves from AC to GVL
+      - Vendor moves from GVL 2 to GVL 3
     """
     tcf_version_hash_model: TCFVersionHash = _build_tcf_version_hash_model(tcf_contents)
     json_str: str = json.dumps(tcf_version_hash_model.dict(), sort_keys=True)

tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py

@@ -824,7 +824,7 @@ def test_tcf_and_ac_enabled_with_overlapping_vendors(
         assert resp.json()["items"][0]["tcf_system_legitimate_interests"] == []
         assert resp.json()["items"][0]["gvl"]["gvlSpecificationVersion"] == 3
         meta = resp.json()["items"][0]["meta"]
-        assert meta["version_hash"] == "f2db7626ca0b"
+        assert meta["version_hash"] == "dbde7265d5dd"
         assert meta["accept_all_fides_string"]
         assert meta["accept_all_fides_string"].endswith(",1~")
         assert meta["accept_all_fides_mobile_data"]