fides: add System model support for new tcf fields

src/fides/api/models/sql_models.py

@@ -399,7 +399,9 @@ class System(Base, FidesBase):
     data_security_practices = Column(String)
     cookie_max_age_seconds = Column(Integer)
     uses_cookies = Column(BOOLEAN(), default=False, server_default="f", nullable=False)
-    cookie_refresh = Column(BOOLEAN(), default=False, server_default="f", nullable=False)
+    cookie_refresh = Column(
+        BOOLEAN(), default=False, server_default="f", nullable=False
+    )
     uses_non_cookie_access = Column(
         BOOLEAN(), default=False, server_default="f", nullable=False
     )

src/fides/api/util/tcf/tcf_experience_contents.py

@@ -426,49 +426,47 @@ def build_purpose_or_feature_section_and_update_vendor_map(
 
 
 def populate_vendor_relationships_basic_attributes(
-    vendor_map,
+    vendor_map: Dict[str, TCFVendorRelationships],
     matching_privacy_declarations: Query,
-):
+) -> Dict[str, TCFVendorRelationships]:
     """Populates TCFVendorRelationships records for all vendors that we're displaying in the overlay.
     Ensures that these TCFVendorRelationships records have the "basic" TCF attributes populated.
     """
     for privacy_declaration_row in matching_privacy_declarations:
         vendor_id, system_identifier = get_system_identifiers(privacy_declaration_row)
-        if vendor_id:  # this only applies to GVL vendors
-            # Add top-level entry to the vendor relationship map
-            if system_identifier not in vendor_map:
-                vendor_map[system_identifier] = TCFVendorRelationships(
-                    id=system_identifier,  # Identify system by vendor id if it exists, otherwise use system id.
-                    name=privacy_declaration_row.system_name,
-                    description=privacy_declaration_row.system_description,
-                    has_vendor_id=bool(
-                        vendor_id
-                    ),  # Has_vendor_id will let us separate data between systems and vendors
-                    cookie_max_age_seconds=privacy_declaration_row.system_cookie_max_age_seconds,
-                    uses_cookies=privacy_declaration_row.system_uses_cookies,
-                    cookie_refresh=privacy_declaration_row.system_cookie_refresh,
-                    uses_non_cookie_access=privacy_declaration_row.system_uses_non_cookie_access,
-                    legitimate_interest_disclosure_url=privacy_declaration_row.system_legitimate_interest_disclosure_url,
-                )
 
-            else:
-                # otherwise just ensure that our basic attributes are populated on the vendor relationship entry
-                vendor_relationships = vendor_map[system_identifier]
-                vendor_relationships.cookie_max_age_seconds = (
-                    privacy_declaration_row.system_cookie_max_age_seconds
-                )
-                vendor_relationships.uses_cookies = (
-                    privacy_declaration_row.system_uses_cookies
-                )
-                vendor_relationships.cookie_refresh = (
-                    privacy_declaration_row.system_cookie_refresh
-                )
-                vendor_relationships.uses_non_cookie_access = (
-                    privacy_declaration_row.system_uses_non_cookie_access
-                )
-                vendor_relationships.legitimate_interest_disclosure_url = (
-                    privacy_declaration_row.system_legitimate_interest_disclosure_url
-                )
+        # Get the existing TCFVendorRelationships record or create a new one.
+        # Add to the vendor map if it wasn't added in a previous section.
+        vendor_relationship_record = vendor_map.get(system_identifier)
+        if not vendor_relationship_record:
+            vendor_relationship_record = TCFVendorRelationships(
+                id=system_identifier,  # Identify system by vendor id if it exists, otherwise use system id.
+                name=privacy_declaration_row.system_name,
+                description=privacy_declaration_row.system_description,
+                has_vendor_id=bool(
+                    vendor_id  # This will let us separate data between systems and vendors later
+                ),
+            )
+            vendor_map[system_identifier] = vendor_relationship_record
+
+        # Now add basic attributes to the VendorRelationships record
+        vendor_relationship_record.cookie_max_age_seconds = (
+            privacy_declaration_row.system_cookie_max_age_seconds
+        )
+        vendor_relationship_record.uses_cookies = (
+            privacy_declaration_row.system_uses_cookies
+        )
+        vendor_relationship_record.cookie_refresh = (
+            privacy_declaration_row.system_cookie_refresh
+        )
+        vendor_relationship_record.uses_non_cookie_access = (
+            privacy_declaration_row.system_uses_non_cookie_access
+        )
+        vendor_relationship_record.legitimate_interest_disclosure_url = (
+            privacy_declaration_row.system_legitimate_interest_disclosure_url
+        )
+
+    return vendor_map
 
 
 def get_tcf_contents(
@@ -564,7 +562,11 @@ def get_tcf_contents(
         matching_privacy_declaration_query=matching_privacy_declarations,
     )
 
-    populate_vendor_relationships_basic_attributes(
+    # Finally, add missing TCFVendorRelationships records for vendors that weren't already added
+    # via the special_features, features, and special_purposes section.  Every vendor in the overlay
+    # should show up in this section. Add the basic attributes to the vendor here to avoid duplication
+    # in other vendor sections.
+    updated_vendor_relationships_map = populate_vendor_relationships_basic_attributes(
         vendor_map=updated_vendor_relationships_map,
         matching_privacy_declarations=matching_privacy_declarations,
     )

tests/ops/util/test_tcf_experience_contents.py

@@ -146,7 +146,7 @@ def test_feature_that_is_not_in_the_gvl(self, db, system):
             v_r_len=0,
             s_c_len=1,
             s_li_len=0,
-            s_r_len=0,
+            s_r_len=1,
         )
 
     def test_system_has_feature_on_different_declaration_than_relevant_use(
@@ -221,6 +221,17 @@ def test_system_has_declaration_no_features_special_features_special_purposes(
             s_r_len=0,
         )
 
+        vendor_relationship = tcf_contents.tcf_vendor_relationships[0]
+        assert vendor_relationship.features == []
+        assert vendor_relationship.special_purposes == []
+        assert vendor_relationship.special_features == []
+        assert vendor_relationship.id == "sendgrid"
+        assert vendor_relationship.cookie_max_age_seconds is None
+        assert vendor_relationship.uses_cookies is False
+        assert vendor_relationship.uses_non_cookie_access is False
+        assert vendor_relationship.cookie_refresh is False
+        assert vendor_relationship.legitimate_interest_disclosure_url is None
+
     @pytest.mark.usefixtures("tcf_system")
     def test_system_exists_with_tcf_purpose_and_vendor(self, db):
         """System has vendor id so we return preferences against a "vendor" instead of the system"""
@@ -687,7 +698,7 @@ def test_system_with_multiple_privacy_declarations(self, db, system):
             v_r_len=0,
             s_c_len=1,
             s_li_len=1,
-            s_r_len=0,
+            s_r_len=1,
         )
 
         first_purpose = tcf_contents.tcf_purpose_consents[0]
@@ -889,7 +900,7 @@ def test_add_same_data_use_to_different_systems(
             v_r_len=0,
             s_c_len=1,
             s_li_len=2,
-            s_r_len=0,
+            s_r_len=2,
         )
         assert len(tcf_contents.tcf_purpose_consents[0].vendors) == 0
         assert tcf_contents.tcf_purpose_consents[0].id == 4