From 12fea7966c3ac10755dcd4938696d0bc6155465f Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 23 Oct 2023 17:59:51 -0500 Subject: [PATCH 1/4] Add a new endpoint - GET Privacy Experience Meta that only returns region, component, and meta info in the response. Only TCF Experiences return any meaningful content at this point. --- .../endpoints/privacy_experience_endpoints.py | 95 ++++++++++++++++--- src/fides/api/schemas/privacy_experience.py | 11 +++ src/fides/common/api/v1/urn_registry.py | 1 + .../test_privacy_experience_endpoints.py | 78 ++++++++++++++- 4 files changed, 173 insertions(+), 12 deletions(-) diff --git a/src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py index 0a4698f0ad..a2a95a34b4 100644 --- a/src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py @@ -25,7 +25,10 @@ ) from fides.api.models.privacy_notice import PrivacyNotice from fides.api.models.privacy_request import ProvidedIdentity -from fides.api.schemas.privacy_experience import PrivacyExperienceResponse +from fides.api.schemas.privacy_experience import ( + PrivacyExperienceMetaResponse, + PrivacyExperienceResponse, +) from fides.api.util.api_router import APIRouter from fides.api.util.consent_util import ( PRIVACY_EXPERIENCE_ESCAPE_FIELDS, @@ -64,6 +67,25 @@ def get_privacy_experience_or_error( return privacy_experience +def _filter_experiences_by_component( + component: ComponentType, experience_query: Query +) -> Query: + """ + Filters privacy experiences by component + + Intentionally relaxes what is returned when querying for "overlay", by returning both types of overlays. + This way the frontend doesn't have to know which type of overlay, regular or tcf, just that it is an overlay. + """ + component_search_map: Dict = { + ComponentType.overlay: [ComponentType.overlay, ComponentType.tcf_overlay] + } + return experience_query.filter( + PrivacyExperience.component.in_( + component_search_map.get(component, [component]) + ) + ) + + def _filter_experiences_by_region_or_country( db: Session, region: Optional[str], experience_query: Query ) -> Query: @@ -119,6 +141,64 @@ def _filter_experiences_by_region_or_country( return db.query(PrivacyExperience).filter(False) +@router.get( + urls.PRIVACY_EXPERIENCE_META, + status_code=HTTP_200_OK, + response_model=Page[PrivacyExperienceMetaResponse], +) +@fides_limiter.limit(CONFIG.security.public_request_rate_limit) +async def get_privacy_experience_meta( + *, + db: Session = Depends(deps.get_db), + params: Params = Depends(), + region: Optional[str] = None, + component: Optional[ComponentType] = None, + request: Request, # required for rate limiting + response: Response, # required for rate limiting +) -> AbstractPage[PrivacyExperienceMetaResponse]: + """Minimal Developer Friendly Privacy Experience endpoint that returns only the meta object, + the component, and the region.""" + + logger.info("Fetching meta info for Experiences '{}'", params) + + await asyncio.sleep(delay=0.001) + experience_query: Query = db.query(PrivacyExperience) + + await asyncio.sleep(delay=0.001) + if region is not None: + experience_query = _filter_experiences_by_region_or_country( + db=db, region=region, experience_query=experience_query + ) + + await asyncio.sleep(delay=0.001) + if component is not None: + experience_query = _filter_experiences_by_component(component, experience_query) + + await asyncio.sleep(delay=0.001) + # TCF contents are the same across all EEA regions, so we can build this once. + base_tcf_contents: TCFExperienceContents = get_tcf_contents(db) + + tcf_meta: Optional[Dict] = None + for ( + tcf_section_name, + _, + ) in TCF_SECTION_MAPPING.items(): + if getattr(base_tcf_contents, tcf_section_name): + # TCF Experience meta is also the same across all EEA regions. Only build meta if there is TCF + # content under at least one section. + tcf_meta = build_experience_tcf_meta(base_tcf_contents) + break + + results: List[PrivacyExperience] = [] + for experience in experience_query: + if experience.component == ComponentType.tcf_overlay: + # Attach meta for TCF Experiences only. We don't yet build meta info for non-TCF experiences. + experience.meta = tcf_meta + results.append(experience) + + return fastapi_paginate(results, params=params) + + @router.get( urls.PRIVACY_EXPERIENCE, status_code=HTTP_200_OK, @@ -196,16 +276,8 @@ async def privacy_experience_list( await asyncio.sleep(delay=0.001) if component is not None: - # Intentionally relaxes what is returned when querying for "overlay", by returning both types of overlays. - # This way the frontend doesn't have to know which type of overlay, regular or tcf, just that it is an overlay. - component_search_map: Dict = { - ComponentType.overlay: [ComponentType.overlay, ComponentType.tcf_overlay] - } - experience_query = experience_query.filter( - PrivacyExperience.component.in_( - component_search_map.get(component, [component]) - ) - ) + experience_query = _filter_experiences_by_component(component, experience_query) + await asyncio.sleep(delay=0.001) if has_config is True: experience_query = experience_query.filter( @@ -221,6 +293,7 @@ async def privacy_experience_list( should_unescape: Optional[str] = request.headers.get(UNESCAPE_SAFESTR_HEADER) # Builds TCF Experience Contents once here, in case multiple TCF Experiences are requested + await asyncio.sleep(delay=0.001) base_tcf_contents: TCFExperienceContents = get_tcf_contents(db) await asyncio.sleep(delay=0.001) diff --git a/src/fides/api/schemas/privacy_experience.py b/src/fides/api/schemas/privacy_experience.py index 1abbe19adf..d5637ad10e 100644 --- a/src/fides/api/schemas/privacy_experience.py +++ b/src/fides/api/schemas/privacy_experience.py @@ -220,3 +220,14 @@ class PrivacyExperienceResponse(TCFExperienceContents, PrivacyExperienceWithId): ) gvl: Optional[Dict] = None meta: Optional[ExperienceMeta] = None + + +class PrivacyExperienceMetaResponse(FidesSchema): + """ + Privacy Experience Response only containing region, component, id, and meta information + """ + + id: str + region: PrivacyNoticeRegion + component: Optional[ComponentType] + meta: Optional[ExperienceMeta] = None diff --git a/src/fides/common/api/v1/urn_registry.py b/src/fides/common/api/v1/urn_registry.py index c315646ee8..251821bcaa 100644 --- a/src/fides/common/api/v1/urn_registry.py +++ b/src/fides/common/api/v1/urn_registry.py @@ -84,6 +84,7 @@ # Privacy Experience URLs PRIVACY_EXPERIENCE = "/privacy-experience" +PRIVACY_EXPERIENCE_META = "/privacy-experience-meta" PRIVACY_EXPERIENCE_DETAIL = "/privacy-experience/{privacy_experience_id}" # Privacy Experience Config URLs diff --git a/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py index ff299b19d3..524989d729 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py @@ -9,7 +9,11 @@ ) from fides.api.models.privacy_experience import ComponentType, PrivacyExperience from fides.api.models.privacy_notice import ConsentMechanism -from fides.common.api.v1.urn_registry import PRIVACY_EXPERIENCE, V1_URL_PREFIX +from fides.common.api.v1.urn_registry import ( + PRIVACY_EXPERIENCE, + PRIVACY_EXPERIENCE_META, + V1_URL_PREFIX, +) class TestGetPrivacyExperiences: @@ -995,3 +999,75 @@ def test_tcf_enabled_but_we_are_not_in_eea(self, db, privacy_experience_overlay) ) assert resp.count() == 1 assert resp.first().id == privacy_experience_overlay.id + + +class TestExperienceMetaEndpoint: + @pytest.fixture(scope="function") + def url(self) -> str: + return V1_URL_PREFIX + PRIVACY_EXPERIENCE_META + + @staticmethod + def assert_no_meta_data(response_json): + assert response_json["meta"]["version_hash"] is None + assert response_json["meta"]["accept_all_fides_string"] is None + assert response_json["meta"]["accept_all_fides_mobile_data"] is None + assert response_json["meta"]["reject_all_fides_string"] is None + assert response_json["meta"]["reject_all_fides_mobile_data"] is None + + @pytest.mark.usefixtures( + "tcf_system", + "privacy_experience_france_overlay", + "privacy_experience_france_tcf_overlay", + ) + def test_tcf_not_enabled(self, url, api_client): + """Regular overlay was returned instead of TCF overlay""" + resp = api_client.get( + url + "?region=fr", + ) + assert resp.status_code == 200 + assert len(resp.json()["items"]) == 1 + resp_json = resp.json()["items"][0] + assert resp_json["component"] == ComponentType.overlay.value + assert resp_json["region"] == "fr" + self.assert_no_meta_data(resp_json) + + @pytest.mark.usefixtures( + "tcf_system", + "privacy_experience_france_overlay", + "enable_tcf", + "privacy_experience_france_tcf_overlay", + ) + def test_tcf_enabled_get_experience_meta(self, url, api_client): + resp = api_client.get( + url + "?region=fr", + ) + assert resp.status_code == 200 + assert len(resp.json()["items"]) == 1 + resp_json = resp.json()["items"][0] + assert resp_json["component"] == ComponentType.tcf_overlay.value + assert resp_json["region"] == "fr" + assert resp_json["meta"]["version_hash"] == "f2db7626ca0b" + assert resp_json["meta"]["accept_all_fides_string"] is not None + assert resp_json["meta"]["accept_all_fides_mobile_data"] is not None + assert resp_json["meta"]["reject_all_fides_string"] is not None + assert resp_json["meta"]["reject_all_fides_mobile_data"] is not None + + @pytest.mark.usefixtures( + "tcf_system", + "privacy_experience_france_overlay", + "enable_tcf", + "privacy_experience_france_tcf_overlay", + "privacy_experience_overlay", + ) + def test_get_ca_experience_meta(self, url, api_client): + """Meta data only built for TCF experiences currently, + so fetching a CA experience will just return null meta details""" + resp = api_client.get( + url + "?region=us_ca", + ) + assert resp.status_code == 200 + assert len(resp.json()["items"]) == 1 + resp_json = resp.json()["items"][0] + assert resp_json["component"] == ComponentType.overlay.value + assert resp_json["region"] == "us_ca" + self.assert_no_meta_data(resp_json) From 965ee50874e1d3a2b593ca9ea1f74c1a4ab9f610 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 23 Oct 2023 18:07:30 -0500 Subject: [PATCH 2/4] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c42fdaff82..ba12ccb3eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ The types of changes are: - Added a `FidesUIChanged` event to Fides.js to track when user preferences change without being saved [#4314](https://github.com/ethyca/fides/pull/4314) and [#4253](https://github.com/ethyca/fides/pull/4253) - Add AC Systems to the TCF Overlay under Vendor Consents section [#4266](https://github.com/ethyca/fides/pull/4266/) - Custom fields are now included in system history change tracking [#4294](https://github.com/ethyca/fides/pull/4294) +- New GET Privacy Experience Meta Endpoint [#4328](https://github.com/ethyca/fides/pull/4328) ### Changed - Derive cookie storage info, privacy policy and legitimate interest disclosure URLs, and data retention data from the data map instead of directly from gvl.json [#4286](https://github.com/ethyca/fides/pull/4286) From 3c88613d820a1b3e3fecf9fd0449ca0b31d66a5d Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 24 Oct 2023 14:05:36 -0500 Subject: [PATCH 3/4] Add new tests for interactions with new behavior now that there's a new AC enabled tag to account for the differences between what is returned in the meta object when AC is enabled versus not enabled --- .../test_privacy_experience_endpoints.py | 86 ++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py index f62b4e9c6d..90d0427119 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py @@ -1199,8 +1199,63 @@ def test_tcf_not_enabled(self, url, api_client): "privacy_experience_france_overlay", "enable_tcf", "privacy_experience_france_tcf_overlay", + "ac_system_with_privacy_declaration", ) def test_tcf_enabled_get_experience_meta(self, url, api_client): + """TCF Enabled but not AC so no AC information is returned""" + resp = api_client.get( + url + "?region=fr", + ) + assert resp.status_code == 200 + assert len(resp.json()["items"]) == 1 + resp_json = resp.json()["items"][0] + assert resp_json["component"] == ComponentType.tcf_overlay.value + assert resp_json["region"] == "fr" + assert resp_json["meta"]["version_hash"] == "dbde7265d5dd" + assert resp_json["meta"]["accept_all_fides_string"] is not None + assert resp_json["meta"]["accept_all_fides_mobile_data"] is not None + + assert ( + resp_json["meta"]["accept_all_fides_string"] + == resp_json["meta"]["accept_all_fides_mobile_data"]["IABTCF_TCString"] + ) + assert ( + resp_json["meta"]["accept_all_fides_mobile_data"]["IABTCF_AddtlConsent"] + is None + ) + assert ( + resp_json["meta"]["accept_all_fides_mobile_data"]["IABTCF_PurposeConsents"] + == "000000010000000000000000" + ) + assert ( + resp_json["meta"]["accept_all_fides_mobile_data"]["IABTCF_VendorConsents"] + == "000000000000000000000000000000000000000001" + ) + assert resp_json["meta"]["reject_all_fides_string"] is not None + assert resp_json["meta"]["reject_all_fides_mobile_data"] is not None + assert ( + resp_json["meta"]["reject_all_fides_mobile_data"]["IABTCF_AddtlConsent"] + is None + ) + assert ( + resp_json["meta"]["reject_all_fides_string"] + == resp_json["meta"]["reject_all_fides_mobile_data"]["IABTCF_TCString"] + ) + + @pytest.mark.usefixtures( + "tcf_system", + "privacy_experience_france_overlay", + "enable_tcf", + "enable_ac", + "privacy_experience_france_tcf_overlay", + "ac_system_with_privacy_declaration", + ) + def test_tcf_and_ac_enabled_get_experience_meta(self, url, api_client): + """TCF Enabled and AC enabled + Purpose consents section is altered because an AC purpose is surfaced in addition to a GVL purpose. + Vendor consents section is the same because AC vendors don't show up in that section. + IABTCF_AddtlConsent is populated + """ resp = api_client.get( url + "?region=fr", ) @@ -1209,11 +1264,40 @@ def test_tcf_enabled_get_experience_meta(self, url, api_client): resp_json = resp.json()["items"][0] assert resp_json["component"] == ComponentType.tcf_overlay.value assert resp_json["region"] == "fr" - assert resp_json["meta"]["version_hash"] == "f2db7626ca0b" + assert resp_json["meta"]["version_hash"] == "ac76b5d026b7" assert resp_json["meta"]["accept_all_fides_string"] is not None assert resp_json["meta"]["accept_all_fides_mobile_data"] is not None + + assert ( + resp_json["meta"]["accept_all_fides_string"] + == resp_json["meta"]["accept_all_fides_mobile_data"]["IABTCF_TCString"] + + "," + + "1~8" + ) + assert ( + resp_json["meta"]["accept_all_fides_mobile_data"]["IABTCF_AddtlConsent"] + == "1~8" + ) + assert ( + resp_json["meta"]["accept_all_fides_mobile_data"]["IABTCF_PurposeConsents"] + == "100000010000000000000000" + ) + assert ( + resp_json["meta"]["accept_all_fides_mobile_data"]["IABTCF_VendorConsents"] + == "000000000000000000000000000000000000000001" + ) assert resp_json["meta"]["reject_all_fides_string"] is not None assert resp_json["meta"]["reject_all_fides_mobile_data"] is not None + assert ( + resp_json["meta"]["reject_all_fides_mobile_data"]["IABTCF_AddtlConsent"] + == "1~" + ) + assert ( + resp_json["meta"]["reject_all_fides_string"] + == resp_json["meta"]["reject_all_fides_mobile_data"]["IABTCF_TCString"] + + "," + + "1~" + ) @pytest.mark.usefixtures( "tcf_system", From b89bccdf0b43339a1e589ae6055b80ecb0c12daf Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 25 Oct 2023 15:37:14 -0500 Subject: [PATCH 4/4] Update return type. --- src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py index a2a95a34b4..d7e0748202 100644 --- a/src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py @@ -155,7 +155,7 @@ async def get_privacy_experience_meta( component: Optional[ComponentType] = None, request: Request, # required for rate limiting response: Response, # required for rate limiting -) -> AbstractPage[PrivacyExperienceMetaResponse]: +) -> AbstractPage[PrivacyExperience]: """Minimal Developer Friendly Privacy Experience endpoint that returns only the meta object, the component, and the region."""